From 4fb23efb1485ccdf4b39fc51eee372b0907b115f Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Fri, 17 May 2019 15:01:02 +0200 Subject: [PATCH 01/22] Move database migrations to Controller --- framework/src/controller/controller.js | 11 +++++++++ framework/src/controller/migrations/index.js | 19 +++++++++++++++ .../migrations}/migration.js | 23 ++++++++----------- .../migrations/sql}/create.sql | 0 .../migrations/sql}/get.sql | 0 .../migrations/sql}/is_persisted.sql | 0 .../updates/20160723182900_create_schema.sql | 0 .../updates/20160723182901_create_views.sql | 0 .../20160724114255_create_memory_tables.sql | 0 ...24132825_upcase_memory_table_addresses.sql | 0 ...60725173858_alter_mem_accounts_columns.sql | 0 ...0022_add_virgin_column_to_mem_accounts.sql | 0 ...908215531_protect_mem_accounts_columns.sql | 0 ...1007153817_create_memory_table_indexes.sql | 0 ...16133824_add_broadhash_column_to_peers.sql | 0 ...0170113181857_add_constraints_to_peers.sql | 0 .../20170124071600_recreate_trs_list_view.sql | 0 .../updates/20170319001337_create_indexes.sql | 0 ...0170321001337_create_rounds_fees_table.sql | 0 .../20170328001337_recreate_trs_list_view.sql | 0 ...0170403001337_calculate_blocks_rewards.sql | 0 .../updates/20170408001337_create_index.sql | 0 ...1337_recreate_calculate_blocks_rewards.sql | 0 .../20170428001337_recreate_trs_lisk_view.sql | 0 ...0614155841_unique_delegates_constraint.sql | 0 ...00_recreate_revert_mem_account_trigger.sql | 0 ...20171207000001_remove_peers_dapp_table.sql | 0 ...171207000006_create_transfer_trs_table.sql | 0 ...07000007_recreate_full_block_list_view.sql | 0 .../20171227155620_rename_port_to_ws_port.sql | 0 .../20180205000000_underscore_patch.sql | 0 ...5000001_drop_rewards_related_functions.sql | 0 ...80205000002_add_height_column_to_peers.sql | 0 ...0205000003_create_rounds_rewards_table.sql | 0 ...05000004_apply_round_exception_mainnet.sql | 0 ...ase_for_blocks_columns_in_mem_accounts.sql | 0 ...20000_enforce_uppercase_trs_recipienId.sql | 0 ...0000_support_long_peer_version_numbers.sql | 0 ...0180419001337_alter_mem_blockid_column.sql | 0 .../20180423001337_remove_virgin_column.sql | 0 ...0503114500_add_row_id_to_trs_list_view.sql | 0 ...180814001337_add_indexes_for_trs_table.sql | 0 ...1001337_recreate_full_blocks_list_view.sql | 0 ...01337_rename_rate_to_rank_in_delegates.sql | 0 ...001337_change_round_type_in_mem_rounds.sql | 0 ...ows_and_add_unique_constraint_in_dapps.sql | 0 ...ows_and_add_unique_constraint_in_votes.sql | 0 ..._and_add_unique_constraint_in_transfer.sql | 0 ...nd_add_unique_constraint_in_intransfer.sql | 0 ...d_unique_constraint_in_multisignatures.sql | 0 ...1106000006_change_wsport_type_in_peers.sql | 0 .../20190103000001_drop_peers_clock.sql | 0 ...ecipient_public_key_to_full_block_list.sql | 0 ...57_add_protocolVersion_column_to_peers.sql | 0 ...ate_trs_dependant_tables_to_trs_trable.sql | 0 ...00_drop_blocks_list_and_trs_list_views.sql | 0 ...0190319111600_remove_unconfirmed_state.sql | 0 ...firmed_state_from_mem_accounts_trigger.sql | 0 ...410112400_add_asset_field_mem_accounts.sql | 0 .../components/storage/entities/index.js | 1 - .../chain/init_steps/bootstrap_storage.js | 3 --- 61 files changed, 40 insertions(+), 17 deletions(-) create mode 100644 framework/src/controller/migrations/index.js rename framework/src/{modules/chain/components/storage/entities => controller/migrations}/migration.js (92%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/create.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/get.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/is_persisted.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20160723182900_create_schema.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20160723182901_create_views.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20160724114255_create_memory_tables.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20160724132825_upcase_memory_table_addresses.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20160725173858_alter_mem_accounts_columns.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20160908120022_add_virgin_column_to_mem_accounts.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20160908215531_protect_mem_accounts_columns.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20161007153817_create_memory_table_indexes.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20161016133824_add_broadhash_column_to_peers.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170113181857_add_constraints_to_peers.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170124071600_recreate_trs_list_view.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170319001337_create_indexes.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170321001337_create_rounds_fees_table.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170328001337_recreate_trs_list_view.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170403001337_calculate_blocks_rewards.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170408001337_create_index.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170422001337_recreate_calculate_blocks_rewards.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170428001337_recreate_trs_lisk_view.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170614155841_unique_delegates_constraint.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20170921105500_recreate_revert_mem_account_trigger.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20171207000001_remove_peers_dapp_table.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20171207000006_create_transfer_trs_table.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20171207000007_recreate_full_block_list_view.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20171227155620_rename_port_to_ws_port.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180205000000_underscore_patch.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180205000001_drop_rewards_related_functions.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180205000002_add_height_column_to_peers.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180205000003_create_rounds_rewards_table.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180205000004_apply_round_exception_mainnet.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180227120000_enforce_uppercase_trs_recipienId.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180327170000_support_long_peer_version_numbers.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180419001337_alter_mem_blockid_column.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180423001337_remove_virgin_column.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180503114500_add_row_id_to_trs_list_view.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180814001337_add_indexes_for_trs_table.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180901001337_recreate_full_blocks_list_view.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20180903001337_rename_rate_to_rank_in_delegates.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20181023001337_change_round_type_in_mem_rounds.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20181106000006_change_wsport_type_in_peers.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20190103000001_drop_peers_clock.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20190104000001_add_recipient_public_key_to_full_block_list.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20190111111557_add_protocolVersion_column_to_peers.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20190313102300_drop_blocks_list_and_trs_list_views.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20190319111600_remove_unconfirmed_state.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql (100%) rename framework/src/{modules/chain/components/storage/sql/migrations => controller/migrations/sql}/updates/20190410112400_add_asset_field_mem_accounts.sql (100%) diff --git a/framework/src/controller/controller.js b/framework/src/controller/controller.js index 7498ce43bf9..2e2537f3881 100644 --- a/framework/src/controller/controller.js +++ b/framework/src/controller/controller.js @@ -24,6 +24,8 @@ const Bus = require('./bus'); const { DuplicateAppInstanceError } = require('../errors'); const { validateModuleSpec } = require('./validator'); const ApplicationState = require('./application_state'); +const { createStorageComponent } = require('../components/storage'); +const { Migration } = require('./migrations'); const isPidRunning = async pid => psList().then(list => list.some(x => x.pid === pid)); @@ -84,6 +86,7 @@ class Controller { await this._validatePidFile(); await this._initState(); await this._setupBus(); + await this._loadMigrations(); await this._loadModules(modules, moduleOptions); this.logger.info('Bus listening to events', this.bus.getEvents()); @@ -189,6 +192,14 @@ class Controller { } } + async _loadMigrations() { + const storageConfig = this.config.components.storage; + this.storage = createStorageComponent(storageConfig, this.logger); + this.storage.registerEntity('Migration', Migration); + await this.storage.bootstrap(); + return this.storage.entities.Migration.applyAll(); + } + async _loadModules(modules, moduleOptions) { // To perform operations in sequence and not using bluebird // eslint-disable-next-line no-restricted-syntax diff --git a/framework/src/controller/migrations/index.js b/framework/src/controller/migrations/index.js new file mode 100644 index 00000000000..6ba7a3c1a41 --- /dev/null +++ b/framework/src/controller/migrations/index.js @@ -0,0 +1,19 @@ +/* + * Copyright © 2018 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + */ + +'use strict'; + +module.exports = { + Migration: require('./migration'), +}; diff --git a/framework/src/modules/chain/components/storage/entities/migration.js b/framework/src/controller/migrations/migration.js similarity index 92% rename from framework/src/modules/chain/components/storage/entities/migration.js rename to framework/src/controller/migrations/migration.js index c84083922b3..abd76f7c55d 100644 --- a/framework/src/modules/chain/components/storage/entities/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -23,14 +23,14 @@ const { utils: { filterTypes: { TEXT }, }, -} = require('../../../../../components/storage'); +} = require('../../components/storage'); const defaultCreateValues = {}; const sqlFiles = { - select: 'migrations/get.sql', - isPersisted: 'migrations/is_persisted.sql', - create: 'migrations/create.sql', + select: 'get.sql', + isPersisted: 'is_persisted.sql', + create: 'create.sql', }; /** @@ -70,8 +70,7 @@ class Migration extends BaseEntity { const defaultSort = { sort: 'id:asc' }; this.extendDefaultOptions(defaultSort); - this.sqlDirectory = path.join(path.dirname(__filename), '../sql'); - + this.sqlDirectory = path.join(path.dirname(__filename), './sql'); this.SQLs = this.loadSQLFiles('migration', sqlFiles, this.sqlDirectory); } @@ -237,14 +236,14 @@ class Migration extends BaseEntity { } /** - * Reads 'sql/migrations/updates' folder and returns an array of objects for further processing. + * Reads/updates' folder and returns an array of objects for further processing. * * @param {number} lastMigrationId * @returns {Promise>} * Promise object that resolves with an array of objects `{id, name, path, file}`. */ readPending(lastMigrationId) { - const updatesPath = path.join(__dirname, '../sql/migrations/updates'); + const updatesPath = `${this.sqlDirectory}/updates`; return fs.readdir(updatesPath).then(files => files .map(migrationFile => { @@ -253,7 +252,7 @@ class Migration extends BaseEntity { migration && { id: migration[1], name: migration[2], - path: path.join('../sql/migrations/updates', migrationFile), + path: path.join(updatesPath, migrationFile), } ); }) @@ -261,13 +260,11 @@ class Migration extends BaseEntity { .filter( migration => migration && - fs - .statSync(path.join(this.sqlDirectory, migration.path)) - .isFile() && + fs.statSync(migration.path).isFile() && (!lastMigrationId || +migration.id > lastMigrationId) ) .map(f => { - f.file = this.adapter.loadSQLFile(f.path, this.sqlDirectory); + f.file = this.adapter.loadSQLFile(f.path, ''); return f; }) ); diff --git a/framework/src/modules/chain/components/storage/sql/migrations/create.sql b/framework/src/controller/migrations/sql/create.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/create.sql rename to framework/src/controller/migrations/sql/create.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/get.sql b/framework/src/controller/migrations/sql/get.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/get.sql rename to framework/src/controller/migrations/sql/get.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/is_persisted.sql b/framework/src/controller/migrations/sql/is_persisted.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/is_persisted.sql rename to framework/src/controller/migrations/sql/is_persisted.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20160723182900_create_schema.sql b/framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20160723182900_create_schema.sql rename to framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20160723182901_create_views.sql b/framework/src/controller/migrations/sql/updates/20160723182901_create_views.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20160723182901_create_views.sql rename to framework/src/controller/migrations/sql/updates/20160723182901_create_views.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20160724114255_create_memory_tables.sql b/framework/src/controller/migrations/sql/updates/20160724114255_create_memory_tables.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20160724114255_create_memory_tables.sql rename to framework/src/controller/migrations/sql/updates/20160724114255_create_memory_tables.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20160724132825_upcase_memory_table_addresses.sql b/framework/src/controller/migrations/sql/updates/20160724132825_upcase_memory_table_addresses.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20160724132825_upcase_memory_table_addresses.sql rename to framework/src/controller/migrations/sql/updates/20160724132825_upcase_memory_table_addresses.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20160725173858_alter_mem_accounts_columns.sql b/framework/src/controller/migrations/sql/updates/20160725173858_alter_mem_accounts_columns.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20160725173858_alter_mem_accounts_columns.sql rename to framework/src/controller/migrations/sql/updates/20160725173858_alter_mem_accounts_columns.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20160908120022_add_virgin_column_to_mem_accounts.sql b/framework/src/controller/migrations/sql/updates/20160908120022_add_virgin_column_to_mem_accounts.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20160908120022_add_virgin_column_to_mem_accounts.sql rename to framework/src/controller/migrations/sql/updates/20160908120022_add_virgin_column_to_mem_accounts.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20160908215531_protect_mem_accounts_columns.sql b/framework/src/controller/migrations/sql/updates/20160908215531_protect_mem_accounts_columns.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20160908215531_protect_mem_accounts_columns.sql rename to framework/src/controller/migrations/sql/updates/20160908215531_protect_mem_accounts_columns.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20161007153817_create_memory_table_indexes.sql b/framework/src/controller/migrations/sql/updates/20161007153817_create_memory_table_indexes.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20161007153817_create_memory_table_indexes.sql rename to framework/src/controller/migrations/sql/updates/20161007153817_create_memory_table_indexes.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20161016133824_add_broadhash_column_to_peers.sql b/framework/src/controller/migrations/sql/updates/20161016133824_add_broadhash_column_to_peers.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20161016133824_add_broadhash_column_to_peers.sql rename to framework/src/controller/migrations/sql/updates/20161016133824_add_broadhash_column_to_peers.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170113181857_add_constraints_to_peers.sql b/framework/src/controller/migrations/sql/updates/20170113181857_add_constraints_to_peers.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170113181857_add_constraints_to_peers.sql rename to framework/src/controller/migrations/sql/updates/20170113181857_add_constraints_to_peers.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170124071600_recreate_trs_list_view.sql b/framework/src/controller/migrations/sql/updates/20170124071600_recreate_trs_list_view.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170124071600_recreate_trs_list_view.sql rename to framework/src/controller/migrations/sql/updates/20170124071600_recreate_trs_list_view.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170319001337_create_indexes.sql b/framework/src/controller/migrations/sql/updates/20170319001337_create_indexes.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170319001337_create_indexes.sql rename to framework/src/controller/migrations/sql/updates/20170319001337_create_indexes.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170321001337_create_rounds_fees_table.sql b/framework/src/controller/migrations/sql/updates/20170321001337_create_rounds_fees_table.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170321001337_create_rounds_fees_table.sql rename to framework/src/controller/migrations/sql/updates/20170321001337_create_rounds_fees_table.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170328001337_recreate_trs_list_view.sql b/framework/src/controller/migrations/sql/updates/20170328001337_recreate_trs_list_view.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170328001337_recreate_trs_list_view.sql rename to framework/src/controller/migrations/sql/updates/20170328001337_recreate_trs_list_view.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170403001337_calculate_blocks_rewards.sql b/framework/src/controller/migrations/sql/updates/20170403001337_calculate_blocks_rewards.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170403001337_calculate_blocks_rewards.sql rename to framework/src/controller/migrations/sql/updates/20170403001337_calculate_blocks_rewards.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170408001337_create_index.sql b/framework/src/controller/migrations/sql/updates/20170408001337_create_index.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170408001337_create_index.sql rename to framework/src/controller/migrations/sql/updates/20170408001337_create_index.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170422001337_recreate_calculate_blocks_rewards.sql b/framework/src/controller/migrations/sql/updates/20170422001337_recreate_calculate_blocks_rewards.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170422001337_recreate_calculate_blocks_rewards.sql rename to framework/src/controller/migrations/sql/updates/20170422001337_recreate_calculate_blocks_rewards.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170428001337_recreate_trs_lisk_view.sql b/framework/src/controller/migrations/sql/updates/20170428001337_recreate_trs_lisk_view.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170428001337_recreate_trs_lisk_view.sql rename to framework/src/controller/migrations/sql/updates/20170428001337_recreate_trs_lisk_view.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170614155841_unique_delegates_constraint.sql b/framework/src/controller/migrations/sql/updates/20170614155841_unique_delegates_constraint.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170614155841_unique_delegates_constraint.sql rename to framework/src/controller/migrations/sql/updates/20170614155841_unique_delegates_constraint.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20170921105500_recreate_revert_mem_account_trigger.sql b/framework/src/controller/migrations/sql/updates/20170921105500_recreate_revert_mem_account_trigger.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20170921105500_recreate_revert_mem_account_trigger.sql rename to framework/src/controller/migrations/sql/updates/20170921105500_recreate_revert_mem_account_trigger.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20171207000001_remove_peers_dapp_table.sql b/framework/src/controller/migrations/sql/updates/20171207000001_remove_peers_dapp_table.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20171207000001_remove_peers_dapp_table.sql rename to framework/src/controller/migrations/sql/updates/20171207000001_remove_peers_dapp_table.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20171207000006_create_transfer_trs_table.sql b/framework/src/controller/migrations/sql/updates/20171207000006_create_transfer_trs_table.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20171207000006_create_transfer_trs_table.sql rename to framework/src/controller/migrations/sql/updates/20171207000006_create_transfer_trs_table.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20171207000007_recreate_full_block_list_view.sql b/framework/src/controller/migrations/sql/updates/20171207000007_recreate_full_block_list_view.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20171207000007_recreate_full_block_list_view.sql rename to framework/src/controller/migrations/sql/updates/20171207000007_recreate_full_block_list_view.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20171227155620_rename_port_to_ws_port.sql b/framework/src/controller/migrations/sql/updates/20171227155620_rename_port_to_ws_port.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20171227155620_rename_port_to_ws_port.sql rename to framework/src/controller/migrations/sql/updates/20171227155620_rename_port_to_ws_port.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180205000000_underscore_patch.sql b/framework/src/controller/migrations/sql/updates/20180205000000_underscore_patch.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180205000000_underscore_patch.sql rename to framework/src/controller/migrations/sql/updates/20180205000000_underscore_patch.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180205000001_drop_rewards_related_functions.sql b/framework/src/controller/migrations/sql/updates/20180205000001_drop_rewards_related_functions.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180205000001_drop_rewards_related_functions.sql rename to framework/src/controller/migrations/sql/updates/20180205000001_drop_rewards_related_functions.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180205000002_add_height_column_to_peers.sql b/framework/src/controller/migrations/sql/updates/20180205000002_add_height_column_to_peers.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180205000002_add_height_column_to_peers.sql rename to framework/src/controller/migrations/sql/updates/20180205000002_add_height_column_to_peers.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180205000003_create_rounds_rewards_table.sql b/framework/src/controller/migrations/sql/updates/20180205000003_create_rounds_rewards_table.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180205000003_create_rounds_rewards_table.sql rename to framework/src/controller/migrations/sql/updates/20180205000003_create_rounds_rewards_table.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180205000004_apply_round_exception_mainnet.sql b/framework/src/controller/migrations/sql/updates/20180205000004_apply_round_exception_mainnet.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180205000004_apply_round_exception_mainnet.sql rename to framework/src/controller/migrations/sql/updates/20180205000004_apply_round_exception_mainnet.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql b/framework/src/controller/migrations/sql/updates/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql rename to framework/src/controller/migrations/sql/updates/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180227120000_enforce_uppercase_trs_recipienId.sql b/framework/src/controller/migrations/sql/updates/20180227120000_enforce_uppercase_trs_recipienId.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180227120000_enforce_uppercase_trs_recipienId.sql rename to framework/src/controller/migrations/sql/updates/20180227120000_enforce_uppercase_trs_recipienId.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180327170000_support_long_peer_version_numbers.sql b/framework/src/controller/migrations/sql/updates/20180327170000_support_long_peer_version_numbers.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180327170000_support_long_peer_version_numbers.sql rename to framework/src/controller/migrations/sql/updates/20180327170000_support_long_peer_version_numbers.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180419001337_alter_mem_blockid_column.sql b/framework/src/controller/migrations/sql/updates/20180419001337_alter_mem_blockid_column.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180419001337_alter_mem_blockid_column.sql rename to framework/src/controller/migrations/sql/updates/20180419001337_alter_mem_blockid_column.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180423001337_remove_virgin_column.sql b/framework/src/controller/migrations/sql/updates/20180423001337_remove_virgin_column.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180423001337_remove_virgin_column.sql rename to framework/src/controller/migrations/sql/updates/20180423001337_remove_virgin_column.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180503114500_add_row_id_to_trs_list_view.sql b/framework/src/controller/migrations/sql/updates/20180503114500_add_row_id_to_trs_list_view.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180503114500_add_row_id_to_trs_list_view.sql rename to framework/src/controller/migrations/sql/updates/20180503114500_add_row_id_to_trs_list_view.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180814001337_add_indexes_for_trs_table.sql b/framework/src/controller/migrations/sql/updates/20180814001337_add_indexes_for_trs_table.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180814001337_add_indexes_for_trs_table.sql rename to framework/src/controller/migrations/sql/updates/20180814001337_add_indexes_for_trs_table.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180901001337_recreate_full_blocks_list_view.sql b/framework/src/controller/migrations/sql/updates/20180901001337_recreate_full_blocks_list_view.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180901001337_recreate_full_blocks_list_view.sql rename to framework/src/controller/migrations/sql/updates/20180901001337_recreate_full_blocks_list_view.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20180903001337_rename_rate_to_rank_in_delegates.sql b/framework/src/controller/migrations/sql/updates/20180903001337_rename_rate_to_rank_in_delegates.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20180903001337_rename_rate_to_rank_in_delegates.sql rename to framework/src/controller/migrations/sql/updates/20180903001337_rename_rate_to_rank_in_delegates.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20181023001337_change_round_type_in_mem_rounds.sql b/framework/src/controller/migrations/sql/updates/20181023001337_change_round_type_in_mem_rounds.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20181023001337_change_round_type_in_mem_rounds.sql rename to framework/src/controller/migrations/sql/updates/20181023001337_change_round_type_in_mem_rounds.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql b/framework/src/controller/migrations/sql/updates/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql rename to framework/src/controller/migrations/sql/updates/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql b/framework/src/controller/migrations/sql/updates/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql rename to framework/src/controller/migrations/sql/updates/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql b/framework/src/controller/migrations/sql/updates/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql rename to framework/src/controller/migrations/sql/updates/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql b/framework/src/controller/migrations/sql/updates/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql rename to framework/src/controller/migrations/sql/updates/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql b/framework/src/controller/migrations/sql/updates/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql rename to framework/src/controller/migrations/sql/updates/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000006_change_wsport_type_in_peers.sql b/framework/src/controller/migrations/sql/updates/20181106000006_change_wsport_type_in_peers.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20181106000006_change_wsport_type_in_peers.sql rename to framework/src/controller/migrations/sql/updates/20181106000006_change_wsport_type_in_peers.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20190103000001_drop_peers_clock.sql b/framework/src/controller/migrations/sql/updates/20190103000001_drop_peers_clock.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20190103000001_drop_peers_clock.sql rename to framework/src/controller/migrations/sql/updates/20190103000001_drop_peers_clock.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20190104000001_add_recipient_public_key_to_full_block_list.sql b/framework/src/controller/migrations/sql/updates/20190104000001_add_recipient_public_key_to_full_block_list.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20190104000001_add_recipient_public_key_to_full_block_list.sql rename to framework/src/controller/migrations/sql/updates/20190104000001_add_recipient_public_key_to_full_block_list.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20190111111557_add_protocolVersion_column_to_peers.sql b/framework/src/controller/migrations/sql/updates/20190111111557_add_protocolVersion_column_to_peers.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20190111111557_add_protocolVersion_column_to_peers.sql rename to framework/src/controller/migrations/sql/updates/20190111111557_add_protocolVersion_column_to_peers.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql b/framework/src/controller/migrations/sql/updates/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql rename to framework/src/controller/migrations/sql/updates/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20190313102300_drop_blocks_list_and_trs_list_views.sql b/framework/src/controller/migrations/sql/updates/20190313102300_drop_blocks_list_and_trs_list_views.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20190313102300_drop_blocks_list_and_trs_list_views.sql rename to framework/src/controller/migrations/sql/updates/20190313102300_drop_blocks_list_and_trs_list_views.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20190319111600_remove_unconfirmed_state.sql b/framework/src/controller/migrations/sql/updates/20190319111600_remove_unconfirmed_state.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20190319111600_remove_unconfirmed_state.sql rename to framework/src/controller/migrations/sql/updates/20190319111600_remove_unconfirmed_state.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql b/framework/src/controller/migrations/sql/updates/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql rename to framework/src/controller/migrations/sql/updates/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql diff --git a/framework/src/modules/chain/components/storage/sql/migrations/updates/20190410112400_add_asset_field_mem_accounts.sql b/framework/src/controller/migrations/sql/updates/20190410112400_add_asset_field_mem_accounts.sql similarity index 100% rename from framework/src/modules/chain/components/storage/sql/migrations/updates/20190410112400_add_asset_field_mem_accounts.sql rename to framework/src/controller/migrations/sql/updates/20190410112400_add_asset_field_mem_accounts.sql diff --git a/framework/src/modules/chain/components/storage/entities/index.js b/framework/src/modules/chain/components/storage/entities/index.js index 2d09ba90950..e405f411396 100644 --- a/framework/src/modules/chain/components/storage/entities/index.js +++ b/framework/src/modules/chain/components/storage/entities/index.js @@ -17,7 +17,6 @@ module.exports = { Account: require('./account'), Block: require('./block'), - Migration: require('./migration'), Round: require('./round'), Transaction: require('./transaction'), }; diff --git a/framework/src/modules/chain/init_steps/bootstrap_storage.js b/framework/src/modules/chain/init_steps/bootstrap_storage.js index 196b3d2b4e5..1eaa3a8677b 100644 --- a/framework/src/modules/chain/init_steps/bootstrap_storage.js +++ b/framework/src/modules/chain/init_steps/bootstrap_storage.js @@ -17,7 +17,6 @@ const { Account, Block, - Migration, Round, Transaction, } = require('../components/storage/entities'); @@ -30,7 +29,6 @@ module.exports = async ({ components: { storage, logger } }, accountLimit) => { storage.registerEntity('Block', Block, { replaceExisting: true, }); - storage.registerEntity('Migration', Migration); storage.registerEntity('Round', Round); storage.registerEntity('Transaction', Transaction, { replaceExisting: true, @@ -42,7 +40,6 @@ module.exports = async ({ components: { storage, logger } }, accountLimit) => { storage.entities.Account.extendDefaultOptions({ limit: accountLimit, }); - await storage.entities.Migration.applyAll(); } catch (err) { logger.error(err); throw err; From 587066656b346d7c1df12c0074126e686d0d3424 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Fri, 17 May 2019 16:54:39 +0200 Subject: [PATCH 02/22] Fix controller jest unit test --- .../test/jest/unit/specs/controller/controller.spec.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/framework/test/jest/unit/specs/controller/controller.spec.js b/framework/test/jest/unit/specs/controller/controller.spec.js index c97c629e2f9..64f3b7afb53 100644 --- a/framework/test/jest/unit/specs/controller/controller.spec.js +++ b/framework/test/jest/unit/specs/controller/controller.spec.js @@ -84,6 +84,9 @@ describe('Controller Class', () => { _validatePidFile: jest.spyOn(controller, '_validatePidFile'), _initState: jest.spyOn(controller, '_initState'), _setupBus: jest.spyOn(controller, '_setupBus'), + _loadMigrations: jest + .spyOn(controller, '_loadMigrations') + .mockImplementation(), _loadModules: jest.spyOn(controller, '_loadModules'), }; const modules = {}; @@ -100,7 +103,8 @@ describe('Controller Class', () => { ); expect(spies._initState).toHaveBeenCalledAfter(spies._validatePidFile); expect(spies._setupBus).toHaveBeenCalledAfter(spies._initState); - expect(spies._loadModules).toHaveBeenCalledAfter(spies._setupBus); + expect(spies._loadMigrations).toHaveBeenCalledAfter(spies._setupBus); + expect(spies._loadModules).toHaveBeenCalledAfter(spies._loadMigrations); expect(spies._loadModules).toHaveBeenCalledWith(modules, moduleOptions); }); @@ -215,6 +219,10 @@ describe('Controller Class', () => { it.todo('should log events if level is greater than info.'); }); + describe('#_loadMigrations', () => { + it.todo('should load migrations.'); + }); + describe('#_loadModules', () => { it('should load modules in sequence', async () => { // Arrange From 377e764c096bd0f68bd01084062eb0ac8674a247 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Mon, 20 May 2019 16:08:26 +0200 Subject: [PATCH 03/22] Separate migrations per modules --- framework/src/controller/application.js | 33 +++- framework/src/controller/controller.js | 23 ++- framework/src/controller/migrations/index.js | 13 +- .../src/controller/migrations/migration.js | 54 +++-- .../sql/20160723182900_create_schema.sql | 26 +++ .../20180205000000_underscore_patch.sql | 0 framework/src/modules/base_module.js | 5 + framework/src/modules/chain/index.js | 5 + .../src/modules/chain/migrations/index.js | 187 ++++++++++++++++++ .../sql/20160723182901_create_schema.sql} | 23 --- .../sql/20160723182903_create_views.sql} | 0 .../20160724114255_create_memory_tables.sql | 0 ...24132825_upcase_memory_table_addresses.sql | 0 ...60725173858_alter_mem_accounts_columns.sql | 0 ...0022_add_virgin_column_to_mem_accounts.sql | 0 ...908215531_protect_mem_accounts_columns.sql | 0 ...1007153817_create_memory_table_indexes.sql | 0 .../20170124071600_recreate_trs_list_view.sql | 0 .../sql}/20170319001337_create_indexes.sql | 0 ...0170321001337_create_rounds_fees_table.sql | 0 .../20170328001337_recreate_trs_list_view.sql | 0 ...0170403001337_calculate_blocks_rewards.sql | 0 .../sql}/20170408001337_create_index.sql | 0 ...1337_recreate_calculate_blocks_rewards.sql | 0 .../20170428001337_recreate_trs_lisk_view.sql | 0 ...0614155841_unique_delegates_constraint.sql | 0 ...00_recreate_revert_mem_account_trigger.sql | 0 ...171207000006_create_transfer_trs_table.sql | 0 ...07000007_recreate_full_block_list_view.sql | 0 ...5000001_drop_rewards_related_functions.sql | 0 ...0205000003_create_rounds_rewards_table.sql | 0 ...05000004_apply_round_exception_mainnet.sql | 0 ...ase_for_blocks_columns_in_mem_accounts.sql | 0 ...20000_enforce_uppercase_trs_recipienId.sql | 0 ...0180419001337_alter_mem_blockid_column.sql | 0 .../20180423001337_remove_virgin_column.sql | 0 ...0503114500_add_row_id_to_trs_list_view.sql | 0 ...180814001337_add_indexes_for_trs_table.sql | 0 ...1001337_recreate_full_blocks_list_view.sql | 0 ...01337_rename_rate_to_rank_in_delegates.sql | 0 ...001337_change_round_type_in_mem_rounds.sql | 0 ...ows_and_add_unique_constraint_in_dapps.sql | 0 ...ows_and_add_unique_constraint_in_votes.sql | 0 ..._and_add_unique_constraint_in_transfer.sql | 0 ...nd_add_unique_constraint_in_intransfer.sql | 0 ...d_unique_constraint_in_multisignatures.sql | 0 ...ecipient_public_key_to_full_block_list.sql | 0 ...ate_trs_dependant_tables_to_trs_trable.sql | 0 ...00_drop_blocks_list_and_trs_list_views.sql | 0 ...0190319111600_remove_unconfirmed_state.sql | 0 ...firmed_state_from_mem_accounts_trigger.sql | 0 ...410112400_add_asset_field_mem_accounts.sql | 0 framework/src/modules/network/index.js | 5 + .../src/modules/network/migrations/index.js | 61 ++++++ .../sql/20160723182902_create_schema.sql | 41 ++++ ...16133824_add_broadhash_column_to_peers.sql | 0 ...0170113181857_add_constraints_to_peers.sql | 0 ...20171207000001_remove_peers_dapp_table.sql | 0 .../20171227155620_rename_port_to_ws_port.sql | 0 ...80205000002_add_height_column_to_peers.sql | 0 ...0000_support_long_peer_version_numbers.sql | 0 ...1106000006_change_wsport_type_in_peers.sql | 0 .../sql}/20190103000001_drop_peers_clock.sql | 0 ...57_add_protocolVersion_column_to_peers.sql | 0 64 files changed, 417 insertions(+), 59 deletions(-) create mode 100644 framework/src/controller/migrations/sql/20160723182900_create_schema.sql rename framework/src/controller/migrations/sql/{updates => }/20180205000000_underscore_patch.sql (100%) create mode 100644 framework/src/modules/chain/migrations/index.js rename framework/src/{controller/migrations/sql/updates/20160723182900_create_schema.sql => modules/chain/migrations/sql/20160723182901_create_schema.sql} (88%) rename framework/src/{controller/migrations/sql/updates/20160723182901_create_views.sql => modules/chain/migrations/sql/20160723182903_create_views.sql} (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20160724114255_create_memory_tables.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20160724132825_upcase_memory_table_addresses.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20160725173858_alter_mem_accounts_columns.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20160908120022_add_virgin_column_to_mem_accounts.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20160908215531_protect_mem_accounts_columns.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20161007153817_create_memory_table_indexes.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20170124071600_recreate_trs_list_view.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20170319001337_create_indexes.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20170321001337_create_rounds_fees_table.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20170328001337_recreate_trs_list_view.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20170403001337_calculate_blocks_rewards.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20170408001337_create_index.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20170422001337_recreate_calculate_blocks_rewards.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20170428001337_recreate_trs_lisk_view.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20170614155841_unique_delegates_constraint.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20170921105500_recreate_revert_mem_account_trigger.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20171207000006_create_transfer_trs_table.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20171207000007_recreate_full_block_list_view.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180205000001_drop_rewards_related_functions.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180205000003_create_rounds_rewards_table.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180205000004_apply_round_exception_mainnet.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180227120000_enforce_uppercase_trs_recipienId.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180419001337_alter_mem_blockid_column.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180423001337_remove_virgin_column.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180503114500_add_row_id_to_trs_list_view.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180814001337_add_indexes_for_trs_table.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180901001337_recreate_full_blocks_list_view.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20180903001337_rename_rate_to_rank_in_delegates.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20181023001337_change_round_type_in_mem_rounds.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20190104000001_add_recipient_public_key_to_full_block_list.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20190313102300_drop_blocks_list_and_trs_list_views.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20190319111600_remove_unconfirmed_state.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/chain/migrations/sql}/20190410112400_add_asset_field_mem_accounts.sql (100%) create mode 100644 framework/src/modules/network/migrations/index.js create mode 100644 framework/src/modules/network/migrations/sql/20160723182902_create_schema.sql rename framework/src/{controller/migrations/sql/updates => modules/network/migrations/sql}/20161016133824_add_broadhash_column_to_peers.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/network/migrations/sql}/20170113181857_add_constraints_to_peers.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/network/migrations/sql}/20171207000001_remove_peers_dapp_table.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/network/migrations/sql}/20171227155620_rename_port_to_ws_port.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/network/migrations/sql}/20180205000002_add_height_column_to_peers.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/network/migrations/sql}/20180327170000_support_long_peer_version_numbers.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/network/migrations/sql}/20181106000006_change_wsport_type_in_peers.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/network/migrations/sql}/20190103000001_drop_peers_clock.sql (100%) rename framework/src/{controller/migrations/sql/updates => modules/network/migrations/sql}/20190111111557_add_protocolVersion_column_to_peers.sql (100%) diff --git a/framework/src/controller/application.js b/framework/src/controller/application.js index 9616fe53281..b4c68838a78 100644 --- a/framework/src/controller/application.js +++ b/framework/src/controller/application.js @@ -40,6 +40,7 @@ const NetworkModule = require('../modules/network'); const __private = { modules: new WeakMap(), transactions: new WeakMap(), + migrations: new WeakMap(), }; const registerProcessHooks = app => { @@ -205,6 +206,9 @@ class Application { options ); __private.modules.set(this, modules); + + // Register migrations defined by the module + this.registerMigrations(moduleKlass.alias, moduleKlass.migrations); } /** @@ -255,6 +259,20 @@ class Application { __private.transactions.set(this, transactions); } + /** + * Register migrations with the application + * + * @param {Object} namespace - Migration namespace + * @param {Array} migrations - Migrations list. Format ['yyyyMMddHHmmss_name_of_migration.sql'] + */ + registerMigrations(namespace, migrations) { + assert(namespace, 'Namespace is required'); + assert(migrations instanceof Array, 'Migrations list should be an array'); + const currentMigrations = this.getMigrations() || {}; + currentMigrations[namespace] = migrations; + __private.migrations.set(this, currentMigrations); + } + /** * Get list of all transactions registered with the application * @@ -293,6 +311,15 @@ class Application { return __private.modules.get(this); } + /** + * Get all registered migrations + * + * @return {Array.} + */ + getMigrations() { + return __private.migrations.get(this); + } + /** * Run the application * @@ -322,7 +349,11 @@ class Application { }, this.logger ); - return this.controller.load(this.getModules(), this.config.modules); + return this.controller.load( + this.getModules(), + this.config.modules, + this.getMigrations() + ); } /** diff --git a/framework/src/controller/controller.js b/framework/src/controller/controller.js index 2e2537f3881..7ed59d565d4 100644 --- a/framework/src/controller/controller.js +++ b/framework/src/controller/controller.js @@ -25,7 +25,10 @@ const { DuplicateAppInstanceError } = require('../errors'); const { validateModuleSpec } = require('./validator'); const ApplicationState = require('./application_state'); const { createStorageComponent } = require('../components/storage'); -const { Migration } = require('./migrations'); +const { + MigrationEntity, + migrations: controllerMigrations, +} = require('./migrations'); const isPidRunning = async pid => psList().then(list => list.some(x => x.pid === pid)); @@ -80,13 +83,13 @@ class Controller { * @param modules * @async */ - async load(modules, moduleOptions) { + async load(modules, moduleOptions, migrations) { this.logger.info('Loading controller'); await this._setupDirectories(); await this._validatePidFile(); await this._initState(); await this._setupBus(); - await this._loadMigrations(); + await this._loadMigrations(migrations); await this._loadModules(modules, moduleOptions); this.logger.info('Bus listening to events', this.bus.getEvents()); @@ -192,12 +195,20 @@ class Controller { } } - async _loadMigrations() { + async _loadMigrations(applicationMigrations) { + const flatten = list => + list.reduce((a, b) => a.concat(Array.isArray(b) ? flatten(b) : b), []); + + const allMigrations = [ + ...controllerMigrations, + ...flatten(Object.values(applicationMigrations)), + ]; + const storageConfig = this.config.components.storage; this.storage = createStorageComponent(storageConfig, this.logger); - this.storage.registerEntity('Migration', Migration); + this.storage.registerEntity('Migration', MigrationEntity); await this.storage.bootstrap(); - return this.storage.entities.Migration.applyAll(); + return this.storage.entities.Migration.apply(allMigrations); } async _loadModules(modules, moduleOptions) { diff --git a/framework/src/controller/migrations/index.js b/framework/src/controller/migrations/index.js index 6ba7a3c1a41..392bc3dc901 100644 --- a/framework/src/controller/migrations/index.js +++ b/framework/src/controller/migrations/index.js @@ -14,6 +14,17 @@ 'use strict'; +const path = require('path'); + +const migrations = [ + path.join(path.dirname(__filename), './sql/20160723182900_create_schema.sql'), + path.join( + path.dirname(__filename), + './sql/20180205000000_underscore_patch.sql' + ), +]; + module.exports = { - Migration: require('./migration'), + MigrationEntity: require('./migration'), + migrations, }; diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration.js index abd76f7c55d..f825b8d32b3 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -242,32 +242,30 @@ class Migration extends BaseEntity { * @returns {Promise>} * Promise object that resolves with an array of objects `{id, name, path, file}`. */ - readPending(lastMigrationId) { - const updatesPath = `${this.sqlDirectory}/updates`; - return fs.readdir(updatesPath).then(files => - files - .map(migrationFile => { - const migration = migrationFile.match(/(\d+)_(.+).sql/); - return ( - migration && { - id: migration[1], - name: migration[2], - path: path.join(updatesPath, migrationFile), - } - ); - }) - .sort((a, b) => a.id - b.id) // Sort by migration ID, ascending - .filter( - migration => - migration && - fs.statSync(migration.path).isFile() && - (!lastMigrationId || +migration.id > lastMigrationId) - ) - .map(f => { - f.file = this.adapter.loadSQLFile(f.path, ''); - return f; - }) - ); + readPending(migrations, lastMigrationId) { + // const updatesPath = `${this.sqlDirectory}/updates`; + return migrations + .map(migrationFile => { + const migration = migrationFile.match(/(\d+)_(.+).sql/); + return ( + migration && { + id: migration[1], + name: migration[2], + path: migrationFile, + } + ); + }) + .sort((a, b) => a.id - b.id) // Sort by migration ID, ascending + .filter( + migration => + migration && + fs.statSync(migration.path).isFile() && + (!lastMigrationId || +migration.id > lastMigrationId) + ) + .map(f => { + f.file = this.adapter.loadSQLFile(f.path, ''); + return f; + }); } async applyPendingMigration(pendingMigration, tx) { @@ -286,10 +284,10 @@ class Migration extends BaseEntity { * * @returns {Promise} Promise object that resolves with `undefined`. */ - async applyAll() { + async apply(migrations) { const hasMigrations = await this.hasMigrations(); const lastId = hasMigrations ? await this.getLastId() : 0; - const pendingMigrations = await this.readPending(lastId); + const pendingMigrations = await this.readPending(migrations, lastId); if (pendingMigrations.length > 0) { // eslint-disable-next-line no-restricted-syntax diff --git a/framework/src/controller/migrations/sql/20160723182900_create_schema.sql b/framework/src/controller/migrations/sql/20160723182900_create_schema.sql new file mode 100644 index 00000000000..7bb943bdb8e --- /dev/null +++ b/framework/src/controller/migrations/sql/20160723182900_create_schema.sql @@ -0,0 +1,26 @@ +/* + * Copyright © 2018 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + */ + + +/* + DESCRIPTION: Creates schema. + + PARAMETERS: None +*/ + +/* Tables */ +CREATE TABLE IF NOT EXISTS "migrations"( + "id" VARCHAR(22) NOT NULL PRIMARY KEY, + "name" TEXT NOT NULL +); diff --git a/framework/src/controller/migrations/sql/updates/20180205000000_underscore_patch.sql b/framework/src/controller/migrations/sql/20180205000000_underscore_patch.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180205000000_underscore_patch.sql rename to framework/src/controller/migrations/sql/20180205000000_underscore_patch.sql diff --git a/framework/src/modules/base_module.js b/framework/src/modules/base_module.js index 3d44d7c4b16..6c2eafbcbb6 100644 --- a/framework/src/modules/base_module.js +++ b/framework/src/modules/base_module.js @@ -31,6 +31,11 @@ module.exports = class BaseModule { throw new ImplementationMissingError(); } + // Array of migrations to be executed before loading the module. Expected format: ['yyyyMMddHHmmss_name_of_migration.sql'] + static get migrations() { + return []; + } + get defaults() { // This interface is not required to be implemented return {}; diff --git a/framework/src/modules/chain/index.js b/framework/src/modules/chain/index.js index 54e9e245b57..f461482b0e3 100644 --- a/framework/src/modules/chain/index.js +++ b/framework/src/modules/chain/index.js @@ -16,6 +16,7 @@ const { config: DefaultConfig } = require('./defaults'); const Chain = require('./chain'); +const { migrations } = require('./migrations'); const BaseModule = require('../base_module'); /* eslint-disable class-methods-use-this */ @@ -45,6 +46,10 @@ module.exports = class ChainModule extends BaseModule { }; } + static get migrations() { + return migrations; + } + static get defaults() { return DefaultConfig; } diff --git a/framework/src/modules/chain/migrations/index.js b/framework/src/modules/chain/migrations/index.js new file mode 100644 index 00000000000..3dad1fdea51 --- /dev/null +++ b/framework/src/modules/chain/migrations/index.js @@ -0,0 +1,187 @@ +/* + * Copyright © 2018 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + */ + +'use strict'; + +const path = require('path'); + +const migrations = [ + path.join(path.dirname(__filename), './sql/20160723182901_create_schema.sql'), + path.join(path.dirname(__filename), './sql/20160723182903_create_views.sql'), + path.join( + path.dirname(__filename), + './sql/20160724114255_create_memory_tables.sql' + ), + path.join( + path.dirname(__filename), + './sql/20160724132825_upcase_memory_table_addresses.sql' + ), + path.join( + path.dirname(__filename), + './sql/20160725173858_alter_mem_accounts_columns.sql' + ), + path.join( + path.dirname(__filename), + './sql/20160908120022_add_virgin_column_to_mem_accounts.sql' + ), + path.join( + path.dirname(__filename), + './sql/20160908215531_protect_mem_accounts_columns.sql' + ), + path.join( + path.dirname(__filename), + './sql/20161007153817_create_memory_table_indexes.sql' + ), + path.join( + path.dirname(__filename), + './sql/20170124071600_recreate_trs_list_view.sql' + ), + path.join( + path.dirname(__filename), + './sql/20170319001337_create_indexes.sql' + ), + path.join( + path.dirname(__filename), + './sql/20170321001337_create_rounds_fees_table.sql' + ), + path.join( + path.dirname(__filename), + './sql/20170328001337_recreate_trs_list_view.sql' + ), + path.join( + path.dirname(__filename), + './sql/20170403001337_calculate_blocks_rewards.sql' + ), + path.join(path.dirname(__filename), './sql/20170408001337_create_index.sql'), + path.join( + path.dirname(__filename), + './sql/20170422001337_recreate_calculate_blocks_rewards.sql' + ), + path.join( + path.dirname(__filename), + './sql/20170428001337_recreate_trs_lisk_view.sql' + ), + path.join( + path.dirname(__filename), + './sql/20170614155841_unique_delegates_constraint.sql' + ), + path.join( + path.dirname(__filename), + './sql/20170921105500_recreate_revert_mem_account_trigger.sql' + ), + path.join( + path.dirname(__filename), + './sql/20171207000006_create_transfer_trs_table.sql' + ), + path.join( + path.dirname(__filename), + './sql/20171207000007_recreate_full_block_list_view.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180205000001_drop_rewards_related_functions.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180205000003_create_rounds_rewards_table.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180205000004_apply_round_exception_mainnet.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180227120000_enforce_uppercase_trs_recipienId.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180419001337_alter_mem_blockid_column.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180423001337_remove_virgin_column.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180503114500_add_row_id_to_trs_list_view.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180814001337_add_indexes_for_trs_table.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180901001337_recreate_full_blocks_list_view.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180903001337_rename_rate_to_rank_in_delegates.sql' + ), + path.join( + path.dirname(__filename), + './sql/20181023001337_change_round_type_in_mem_rounds.sql' + ), + path.join( + path.dirname(__filename), + './sql/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql' + ), + path.join( + path.dirname(__filename), + './sql/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql' + ), + path.join( + path.dirname(__filename), + './sql/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql' + ), + path.join( + path.dirname(__filename), + './sql/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql' + ), + path.join( + path.dirname(__filename), + './sql/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql' + ), + path.join( + path.dirname(__filename), + './sql/20190104000001_add_recipient_public_key_to_full_block_list.sql' + ), + path.join( + path.dirname(__filename), + './sql/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql' + ), + path.join( + path.dirname(__filename), + './sql/20190313102300_drop_blocks_list_and_trs_list_views.sql' + ), + path.join( + path.dirname(__filename), + './sql/20190319111600_remove_unconfirmed_state.sql' + ), + path.join( + path.dirname(__filename), + './sql/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql' + ), + path.join( + path.dirname(__filename), + './sql/20190410112400_add_asset_field_mem_accounts.sql' + ), +]; + +module.exports = { + migrations, +}; diff --git a/framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql b/framework/src/modules/chain/migrations/sql/20160723182901_create_schema.sql similarity index 88% rename from framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql rename to framework/src/modules/chain/migrations/sql/20160723182901_create_schema.sql index c0275b016e3..8fbe9d0b571 100644 --- a/framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql +++ b/framework/src/modules/chain/migrations/sql/20160723182901_create_schema.sql @@ -20,11 +20,6 @@ */ /* Tables */ -CREATE TABLE IF NOT EXISTS "migrations"( - "id" VARCHAR(22) NOT NULL PRIMARY KEY, - "name" TEXT NOT NULL -); - CREATE TABLE IF NOT EXISTS "blocks"( "id" VARCHAR(20) PRIMARY KEY, "rowId" SERIAL NOT NULL, @@ -122,28 +117,10 @@ CREATE TABLE IF NOT EXISTS "outtransfer"( FOREIGN KEY("transactionId") REFERENCES "trs"("id") ON DELETE CASCADE ); -CREATE TABLE IF NOT EXISTS "peers"( - "id" SERIAL NOT NULL PRIMARY KEY, - "ip" INET NOT NULL, - "port" SMALLINT NOT NULL, - "state" SMALLINT NOT NULL, - "os" VARCHAR(64), - "version" VARCHAR(11), - "clock" BIGINT -); - -CREATE TABLE IF NOT EXISTS "peers_dapp"( - "peerId" INT NOT NULL, - "dappid" VARCHAR(20) NOT NULL, - FOREIGN KEY("peerId") REFERENCES "peers"("id") ON DELETE CASCADE -); - /* Unique Indexes */ CREATE UNIQUE INDEX IF NOT EXISTS "blocks_height" ON "blocks"("height"); CREATE UNIQUE INDEX IF NOT EXISTS "blocks_previousBlock" ON "blocks"("previousBlock"); CREATE UNIQUE INDEX IF Not EXISTS "out_transaction_id" ON "outtransfer"("outTransactionId"); -CREATE UNIQUE INDEX IF NOT EXISTS "peers_unique" ON "peers"("ip", "port"); -CREATE UNIQUE INDEX IF NOT EXISTS "peers_dapp_unique" ON "peers_dapp"("peerId", "dappid"); /* Indexes */ CREATE INDEX IF NOT EXISTS "blocks_rowId" ON "blocks"("rowId"); diff --git a/framework/src/controller/migrations/sql/updates/20160723182901_create_views.sql b/framework/src/modules/chain/migrations/sql/20160723182903_create_views.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20160723182901_create_views.sql rename to framework/src/modules/chain/migrations/sql/20160723182903_create_views.sql diff --git a/framework/src/controller/migrations/sql/updates/20160724114255_create_memory_tables.sql b/framework/src/modules/chain/migrations/sql/20160724114255_create_memory_tables.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20160724114255_create_memory_tables.sql rename to framework/src/modules/chain/migrations/sql/20160724114255_create_memory_tables.sql diff --git a/framework/src/controller/migrations/sql/updates/20160724132825_upcase_memory_table_addresses.sql b/framework/src/modules/chain/migrations/sql/20160724132825_upcase_memory_table_addresses.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20160724132825_upcase_memory_table_addresses.sql rename to framework/src/modules/chain/migrations/sql/20160724132825_upcase_memory_table_addresses.sql diff --git a/framework/src/controller/migrations/sql/updates/20160725173858_alter_mem_accounts_columns.sql b/framework/src/modules/chain/migrations/sql/20160725173858_alter_mem_accounts_columns.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20160725173858_alter_mem_accounts_columns.sql rename to framework/src/modules/chain/migrations/sql/20160725173858_alter_mem_accounts_columns.sql diff --git a/framework/src/controller/migrations/sql/updates/20160908120022_add_virgin_column_to_mem_accounts.sql b/framework/src/modules/chain/migrations/sql/20160908120022_add_virgin_column_to_mem_accounts.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20160908120022_add_virgin_column_to_mem_accounts.sql rename to framework/src/modules/chain/migrations/sql/20160908120022_add_virgin_column_to_mem_accounts.sql diff --git a/framework/src/controller/migrations/sql/updates/20160908215531_protect_mem_accounts_columns.sql b/framework/src/modules/chain/migrations/sql/20160908215531_protect_mem_accounts_columns.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20160908215531_protect_mem_accounts_columns.sql rename to framework/src/modules/chain/migrations/sql/20160908215531_protect_mem_accounts_columns.sql diff --git a/framework/src/controller/migrations/sql/updates/20161007153817_create_memory_table_indexes.sql b/framework/src/modules/chain/migrations/sql/20161007153817_create_memory_table_indexes.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20161007153817_create_memory_table_indexes.sql rename to framework/src/modules/chain/migrations/sql/20161007153817_create_memory_table_indexes.sql diff --git a/framework/src/controller/migrations/sql/updates/20170124071600_recreate_trs_list_view.sql b/framework/src/modules/chain/migrations/sql/20170124071600_recreate_trs_list_view.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170124071600_recreate_trs_list_view.sql rename to framework/src/modules/chain/migrations/sql/20170124071600_recreate_trs_list_view.sql diff --git a/framework/src/controller/migrations/sql/updates/20170319001337_create_indexes.sql b/framework/src/modules/chain/migrations/sql/20170319001337_create_indexes.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170319001337_create_indexes.sql rename to framework/src/modules/chain/migrations/sql/20170319001337_create_indexes.sql diff --git a/framework/src/controller/migrations/sql/updates/20170321001337_create_rounds_fees_table.sql b/framework/src/modules/chain/migrations/sql/20170321001337_create_rounds_fees_table.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170321001337_create_rounds_fees_table.sql rename to framework/src/modules/chain/migrations/sql/20170321001337_create_rounds_fees_table.sql diff --git a/framework/src/controller/migrations/sql/updates/20170328001337_recreate_trs_list_view.sql b/framework/src/modules/chain/migrations/sql/20170328001337_recreate_trs_list_view.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170328001337_recreate_trs_list_view.sql rename to framework/src/modules/chain/migrations/sql/20170328001337_recreate_trs_list_view.sql diff --git a/framework/src/controller/migrations/sql/updates/20170403001337_calculate_blocks_rewards.sql b/framework/src/modules/chain/migrations/sql/20170403001337_calculate_blocks_rewards.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170403001337_calculate_blocks_rewards.sql rename to framework/src/modules/chain/migrations/sql/20170403001337_calculate_blocks_rewards.sql diff --git a/framework/src/controller/migrations/sql/updates/20170408001337_create_index.sql b/framework/src/modules/chain/migrations/sql/20170408001337_create_index.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170408001337_create_index.sql rename to framework/src/modules/chain/migrations/sql/20170408001337_create_index.sql diff --git a/framework/src/controller/migrations/sql/updates/20170422001337_recreate_calculate_blocks_rewards.sql b/framework/src/modules/chain/migrations/sql/20170422001337_recreate_calculate_blocks_rewards.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170422001337_recreate_calculate_blocks_rewards.sql rename to framework/src/modules/chain/migrations/sql/20170422001337_recreate_calculate_blocks_rewards.sql diff --git a/framework/src/controller/migrations/sql/updates/20170428001337_recreate_trs_lisk_view.sql b/framework/src/modules/chain/migrations/sql/20170428001337_recreate_trs_lisk_view.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170428001337_recreate_trs_lisk_view.sql rename to framework/src/modules/chain/migrations/sql/20170428001337_recreate_trs_lisk_view.sql diff --git a/framework/src/controller/migrations/sql/updates/20170614155841_unique_delegates_constraint.sql b/framework/src/modules/chain/migrations/sql/20170614155841_unique_delegates_constraint.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170614155841_unique_delegates_constraint.sql rename to framework/src/modules/chain/migrations/sql/20170614155841_unique_delegates_constraint.sql diff --git a/framework/src/controller/migrations/sql/updates/20170921105500_recreate_revert_mem_account_trigger.sql b/framework/src/modules/chain/migrations/sql/20170921105500_recreate_revert_mem_account_trigger.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170921105500_recreate_revert_mem_account_trigger.sql rename to framework/src/modules/chain/migrations/sql/20170921105500_recreate_revert_mem_account_trigger.sql diff --git a/framework/src/controller/migrations/sql/updates/20171207000006_create_transfer_trs_table.sql b/framework/src/modules/chain/migrations/sql/20171207000006_create_transfer_trs_table.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20171207000006_create_transfer_trs_table.sql rename to framework/src/modules/chain/migrations/sql/20171207000006_create_transfer_trs_table.sql diff --git a/framework/src/controller/migrations/sql/updates/20171207000007_recreate_full_block_list_view.sql b/framework/src/modules/chain/migrations/sql/20171207000007_recreate_full_block_list_view.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20171207000007_recreate_full_block_list_view.sql rename to framework/src/modules/chain/migrations/sql/20171207000007_recreate_full_block_list_view.sql diff --git a/framework/src/controller/migrations/sql/updates/20180205000001_drop_rewards_related_functions.sql b/framework/src/modules/chain/migrations/sql/20180205000001_drop_rewards_related_functions.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180205000001_drop_rewards_related_functions.sql rename to framework/src/modules/chain/migrations/sql/20180205000001_drop_rewards_related_functions.sql diff --git a/framework/src/controller/migrations/sql/updates/20180205000003_create_rounds_rewards_table.sql b/framework/src/modules/chain/migrations/sql/20180205000003_create_rounds_rewards_table.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180205000003_create_rounds_rewards_table.sql rename to framework/src/modules/chain/migrations/sql/20180205000003_create_rounds_rewards_table.sql diff --git a/framework/src/controller/migrations/sql/updates/20180205000004_apply_round_exception_mainnet.sql b/framework/src/modules/chain/migrations/sql/20180205000004_apply_round_exception_mainnet.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180205000004_apply_round_exception_mainnet.sql rename to framework/src/modules/chain/migrations/sql/20180205000004_apply_round_exception_mainnet.sql diff --git a/framework/src/controller/migrations/sql/updates/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql b/framework/src/modules/chain/migrations/sql/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql rename to framework/src/modules/chain/migrations/sql/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql diff --git a/framework/src/controller/migrations/sql/updates/20180227120000_enforce_uppercase_trs_recipienId.sql b/framework/src/modules/chain/migrations/sql/20180227120000_enforce_uppercase_trs_recipienId.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180227120000_enforce_uppercase_trs_recipienId.sql rename to framework/src/modules/chain/migrations/sql/20180227120000_enforce_uppercase_trs_recipienId.sql diff --git a/framework/src/controller/migrations/sql/updates/20180419001337_alter_mem_blockid_column.sql b/framework/src/modules/chain/migrations/sql/20180419001337_alter_mem_blockid_column.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180419001337_alter_mem_blockid_column.sql rename to framework/src/modules/chain/migrations/sql/20180419001337_alter_mem_blockid_column.sql diff --git a/framework/src/controller/migrations/sql/updates/20180423001337_remove_virgin_column.sql b/framework/src/modules/chain/migrations/sql/20180423001337_remove_virgin_column.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180423001337_remove_virgin_column.sql rename to framework/src/modules/chain/migrations/sql/20180423001337_remove_virgin_column.sql diff --git a/framework/src/controller/migrations/sql/updates/20180503114500_add_row_id_to_trs_list_view.sql b/framework/src/modules/chain/migrations/sql/20180503114500_add_row_id_to_trs_list_view.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180503114500_add_row_id_to_trs_list_view.sql rename to framework/src/modules/chain/migrations/sql/20180503114500_add_row_id_to_trs_list_view.sql diff --git a/framework/src/controller/migrations/sql/updates/20180814001337_add_indexes_for_trs_table.sql b/framework/src/modules/chain/migrations/sql/20180814001337_add_indexes_for_trs_table.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180814001337_add_indexes_for_trs_table.sql rename to framework/src/modules/chain/migrations/sql/20180814001337_add_indexes_for_trs_table.sql diff --git a/framework/src/controller/migrations/sql/updates/20180901001337_recreate_full_blocks_list_view.sql b/framework/src/modules/chain/migrations/sql/20180901001337_recreate_full_blocks_list_view.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180901001337_recreate_full_blocks_list_view.sql rename to framework/src/modules/chain/migrations/sql/20180901001337_recreate_full_blocks_list_view.sql diff --git a/framework/src/controller/migrations/sql/updates/20180903001337_rename_rate_to_rank_in_delegates.sql b/framework/src/modules/chain/migrations/sql/20180903001337_rename_rate_to_rank_in_delegates.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180903001337_rename_rate_to_rank_in_delegates.sql rename to framework/src/modules/chain/migrations/sql/20180903001337_rename_rate_to_rank_in_delegates.sql diff --git a/framework/src/controller/migrations/sql/updates/20181023001337_change_round_type_in_mem_rounds.sql b/framework/src/modules/chain/migrations/sql/20181023001337_change_round_type_in_mem_rounds.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20181023001337_change_round_type_in_mem_rounds.sql rename to framework/src/modules/chain/migrations/sql/20181023001337_change_round_type_in_mem_rounds.sql diff --git a/framework/src/controller/migrations/sql/updates/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql b/framework/src/modules/chain/migrations/sql/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql rename to framework/src/modules/chain/migrations/sql/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql diff --git a/framework/src/controller/migrations/sql/updates/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql b/framework/src/modules/chain/migrations/sql/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql rename to framework/src/modules/chain/migrations/sql/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql diff --git a/framework/src/controller/migrations/sql/updates/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql b/framework/src/modules/chain/migrations/sql/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql rename to framework/src/modules/chain/migrations/sql/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql diff --git a/framework/src/controller/migrations/sql/updates/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql b/framework/src/modules/chain/migrations/sql/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql rename to framework/src/modules/chain/migrations/sql/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql diff --git a/framework/src/controller/migrations/sql/updates/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql b/framework/src/modules/chain/migrations/sql/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql rename to framework/src/modules/chain/migrations/sql/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql diff --git a/framework/src/controller/migrations/sql/updates/20190104000001_add_recipient_public_key_to_full_block_list.sql b/framework/src/modules/chain/migrations/sql/20190104000001_add_recipient_public_key_to_full_block_list.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20190104000001_add_recipient_public_key_to_full_block_list.sql rename to framework/src/modules/chain/migrations/sql/20190104000001_add_recipient_public_key_to_full_block_list.sql diff --git a/framework/src/controller/migrations/sql/updates/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql b/framework/src/modules/chain/migrations/sql/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql rename to framework/src/modules/chain/migrations/sql/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql diff --git a/framework/src/controller/migrations/sql/updates/20190313102300_drop_blocks_list_and_trs_list_views.sql b/framework/src/modules/chain/migrations/sql/20190313102300_drop_blocks_list_and_trs_list_views.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20190313102300_drop_blocks_list_and_trs_list_views.sql rename to framework/src/modules/chain/migrations/sql/20190313102300_drop_blocks_list_and_trs_list_views.sql diff --git a/framework/src/controller/migrations/sql/updates/20190319111600_remove_unconfirmed_state.sql b/framework/src/modules/chain/migrations/sql/20190319111600_remove_unconfirmed_state.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20190319111600_remove_unconfirmed_state.sql rename to framework/src/modules/chain/migrations/sql/20190319111600_remove_unconfirmed_state.sql diff --git a/framework/src/controller/migrations/sql/updates/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql b/framework/src/modules/chain/migrations/sql/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql rename to framework/src/modules/chain/migrations/sql/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql diff --git a/framework/src/controller/migrations/sql/updates/20190410112400_add_asset_field_mem_accounts.sql b/framework/src/modules/chain/migrations/sql/20190410112400_add_asset_field_mem_accounts.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20190410112400_add_asset_field_mem_accounts.sql rename to framework/src/modules/chain/migrations/sql/20190410112400_add_asset_field_mem_accounts.sql diff --git a/framework/src/modules/network/index.js b/framework/src/modules/network/index.js index 506b70aa5a5..da9aed8bc19 100644 --- a/framework/src/modules/network/index.js +++ b/framework/src/modules/network/index.js @@ -15,6 +15,7 @@ 'use strict'; const { config } = require('./defaults'); +const { migrations } = require('./migrations'); const Network = require('./network'); const BaseModule = require('../base_module'); @@ -39,6 +40,10 @@ module.exports = class NetworkModule extends BaseModule { }; } + static get migrations() { + return migrations; + } + static get defaults() { return config; } diff --git a/framework/src/modules/network/migrations/index.js b/framework/src/modules/network/migrations/index.js new file mode 100644 index 00000000000..999b2a958a8 --- /dev/null +++ b/framework/src/modules/network/migrations/index.js @@ -0,0 +1,61 @@ +/* + * Copyright © 2018 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + */ + +'use strict'; + +const path = require('path'); + +const migrations = [ + path.join(path.dirname(__filename), './sql/20160723182902_create_schema.sql'), + path.join( + path.dirname(__filename), + './sql/20161016133824_add_broadhash_column_to_peers.sql' + ), + path.join( + path.dirname(__filename), + './sql/20170113181857_add_constraints_to_peers.sql' + ), + path.join( + path.dirname(__filename), + './sql/20171207000001_remove_peers_dapp_table.sql' + ), + path.join( + path.dirname(__filename), + './sql/20171227155620_rename_port_to_ws_port.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180205000002_add_height_column_to_peers.sql' + ), + path.join( + path.dirname(__filename), + './sql/20180327170000_support_long_peer_version_numbers.sql' + ), + path.join( + path.dirname(__filename), + './sql/20181106000006_change_wsport_type_in_peers.sql' + ), + path.join( + path.dirname(__filename), + './sql/20190103000001_drop_peers_clock.sql' + ), + path.join( + path.dirname(__filename), + './sql/20190111111557_add_protocolVersion_column_to_peers.sql' + ), +]; + +module.exports = { + migrations, +}; diff --git a/framework/src/modules/network/migrations/sql/20160723182902_create_schema.sql b/framework/src/modules/network/migrations/sql/20160723182902_create_schema.sql new file mode 100644 index 00000000000..febb3a5ad9e --- /dev/null +++ b/framework/src/modules/network/migrations/sql/20160723182902_create_schema.sql @@ -0,0 +1,41 @@ +/* + * Copyright © 2018 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + */ + + +/* + DESCRIPTION: Creates schema. + + PARAMETERS: None +*/ + +/* Tables */ +CREATE TABLE IF NOT EXISTS "peers"( + "id" SERIAL NOT NULL PRIMARY KEY, + "ip" INET NOT NULL, + "port" SMALLINT NOT NULL, + "state" SMALLINT NOT NULL, + "os" VARCHAR(64), + "version" VARCHAR(11), + "clock" BIGINT +); + +CREATE TABLE IF NOT EXISTS "peers_dapp"( + "peerId" INT NOT NULL, + "dappid" VARCHAR(20) NOT NULL, + FOREIGN KEY("peerId") REFERENCES "peers"("id") ON DELETE CASCADE +); + +/* Unique Indexes */ +CREATE UNIQUE INDEX IF NOT EXISTS "peers_unique" ON "peers"("ip", "port"); +CREATE UNIQUE INDEX IF NOT EXISTS "peers_dapp_unique" ON "peers_dapp"("peerId", "dappid"); diff --git a/framework/src/controller/migrations/sql/updates/20161016133824_add_broadhash_column_to_peers.sql b/framework/src/modules/network/migrations/sql/20161016133824_add_broadhash_column_to_peers.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20161016133824_add_broadhash_column_to_peers.sql rename to framework/src/modules/network/migrations/sql/20161016133824_add_broadhash_column_to_peers.sql diff --git a/framework/src/controller/migrations/sql/updates/20170113181857_add_constraints_to_peers.sql b/framework/src/modules/network/migrations/sql/20170113181857_add_constraints_to_peers.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20170113181857_add_constraints_to_peers.sql rename to framework/src/modules/network/migrations/sql/20170113181857_add_constraints_to_peers.sql diff --git a/framework/src/controller/migrations/sql/updates/20171207000001_remove_peers_dapp_table.sql b/framework/src/modules/network/migrations/sql/20171207000001_remove_peers_dapp_table.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20171207000001_remove_peers_dapp_table.sql rename to framework/src/modules/network/migrations/sql/20171207000001_remove_peers_dapp_table.sql diff --git a/framework/src/controller/migrations/sql/updates/20171227155620_rename_port_to_ws_port.sql b/framework/src/modules/network/migrations/sql/20171227155620_rename_port_to_ws_port.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20171227155620_rename_port_to_ws_port.sql rename to framework/src/modules/network/migrations/sql/20171227155620_rename_port_to_ws_port.sql diff --git a/framework/src/controller/migrations/sql/updates/20180205000002_add_height_column_to_peers.sql b/framework/src/modules/network/migrations/sql/20180205000002_add_height_column_to_peers.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180205000002_add_height_column_to_peers.sql rename to framework/src/modules/network/migrations/sql/20180205000002_add_height_column_to_peers.sql diff --git a/framework/src/controller/migrations/sql/updates/20180327170000_support_long_peer_version_numbers.sql b/framework/src/modules/network/migrations/sql/20180327170000_support_long_peer_version_numbers.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20180327170000_support_long_peer_version_numbers.sql rename to framework/src/modules/network/migrations/sql/20180327170000_support_long_peer_version_numbers.sql diff --git a/framework/src/controller/migrations/sql/updates/20181106000006_change_wsport_type_in_peers.sql b/framework/src/modules/network/migrations/sql/20181106000006_change_wsport_type_in_peers.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20181106000006_change_wsport_type_in_peers.sql rename to framework/src/modules/network/migrations/sql/20181106000006_change_wsport_type_in_peers.sql diff --git a/framework/src/controller/migrations/sql/updates/20190103000001_drop_peers_clock.sql b/framework/src/modules/network/migrations/sql/20190103000001_drop_peers_clock.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20190103000001_drop_peers_clock.sql rename to framework/src/modules/network/migrations/sql/20190103000001_drop_peers_clock.sql diff --git a/framework/src/controller/migrations/sql/updates/20190111111557_add_protocolVersion_column_to_peers.sql b/framework/src/modules/network/migrations/sql/20190111111557_add_protocolVersion_column_to_peers.sql similarity index 100% rename from framework/src/controller/migrations/sql/updates/20190111111557_add_protocolVersion_column_to_peers.sql rename to framework/src/modules/network/migrations/sql/20190111111557_add_protocolVersion_column_to_peers.sql From f20207ba03977b0ff2ca108135b2159c7a5a4737 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Mon, 20 May 2019 18:31:03 +0200 Subject: [PATCH 04/22] Execute all migrations not yet executed rather than by ID --- framework/src/controller/controller.js | 18 +++--- .../src/controller/migrations/migration.js | 62 +++++++++++-------- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/framework/src/controller/controller.js b/framework/src/controller/controller.js index 7ed59d565d4..5af42a63384 100644 --- a/framework/src/controller/controller.js +++ b/framework/src/controller/controller.js @@ -27,7 +27,7 @@ const ApplicationState = require('./application_state'); const { createStorageComponent } = require('../components/storage'); const { MigrationEntity, - migrations: controllerMigrations, + migrations: frameworkMigrations, } = require('./migrations'); const isPidRunning = async pid => @@ -195,20 +195,18 @@ class Controller { } } - async _loadMigrations(applicationMigrations) { - const flatten = list => - list.reduce((a, b) => a.concat(Array.isArray(b) ? flatten(b) : b), []); - - const allMigrations = [ - ...controllerMigrations, - ...flatten(Object.values(applicationMigrations)), - ]; + async _loadMigrations(applicationMigrationsObj) { + const frameworkMigrationsObj = { + framework: frameworkMigrations, + }; const storageConfig = this.config.components.storage; this.storage = createStorageComponent(storageConfig, this.logger); this.storage.registerEntity('Migration', MigrationEntity); await this.storage.bootstrap(); - return this.storage.entities.Migration.apply(allMigrations); + await this.storage.entities.Migration.apply(frameworkMigrationsObj); + await this.storage.entities.Migration.apply(applicationMigrationsObj); + return true; } async _loadModules(modules, moduleOptions) { diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration.js index f825b8d32b3..3c28cdf5503 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -242,34 +242,37 @@ class Migration extends BaseEntity { * @returns {Promise>} * Promise object that resolves with an array of objects `{id, name, path, file}`. */ - readPending(migrations, lastMigrationId) { + async readPending(migrationsObj, executedMigrationIDs) { // const updatesPath = `${this.sqlDirectory}/updates`; - return migrations - .map(migrationFile => { - const migration = migrationFile.match(/(\d+)_(.+).sql/); - return ( - migration && { - id: migration[1], - name: migration[2], - path: migrationFile, - } - ); - }) - .sort((a, b) => a.id - b.id) // Sort by migration ID, ascending - .filter( - migration => - migration && - fs.statSync(migration.path).isFile() && - (!lastMigrationId || +migration.id > lastMigrationId) - ) - .map(f => { - f.file = this.adapter.loadSQLFile(f.path, ''); - return f; - }); + return Object.keys(migrationsObj).reduce((prev, namespace) => { + const curr = migrationsObj[namespace] + .map(migrationFile => { + const migration = migrationFile.match(/(\d+)_(.+).sql/); + return ( + migration && { + id: migration[1], + name: migration[2], + path: migrationFile, + namespace, + } + ); + }) + .sort((a, b) => a.id - b.id) // Sort by migration ID, ascending + .filter( + migration => + migration && + fs.statSync(migration.path).isFile() && + !executedMigrationIDs.includes(migration.id) + ) + .map(migration => { + migration.file = this.adapter.loadSQLFile(migration.path, ''); + return migration; + }); + return prev.concat(curr); + }, []); } async applyPendingMigration(pendingMigration, tx) { - // eslint-disable-next-line no-restricted-syntax await this.adapter.executeFile(pendingMigration.file, {}, {}, tx); await this.create( { id: pendingMigration.id, name: pendingMigration.name }, @@ -284,10 +287,15 @@ class Migration extends BaseEntity { * * @returns {Promise} Promise object that resolves with `undefined`. */ - async apply(migrations) { + async apply(migrationsObj) { const hasMigrations = await this.hasMigrations(); - const lastId = hasMigrations ? await this.getLastId() : 0; - const pendingMigrations = await this.readPending(migrations, lastId); + const executedMigrationIDs = hasMigrations + ? (await this.get({}, { limit: null })).map(m => m.id) + : []; + const pendingMigrations = await this.readPending( + migrationsObj, + executedMigrationIDs + ); if (pendingMigrations.length > 0) { // eslint-disable-next-line no-restricted-syntax From 60877bfefbbaf5070168477ae7ae7ab8e6bfc256 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Tue, 21 May 2019 14:15:15 +0200 Subject: [PATCH 05/22] Implement internal_migrations table Separate framework migrations from app/modules migrations in order to be able to migrate code and database all together --- framework/src/controller/controller.js | 13 +-- .../src/controller/migrations/migration.js | 110 ++++++++++++++++-- .../src/controller/migrations/sql/get.sql | 3 +- .../20160723182900_create_schema.sql | 8 +- .../20180205000000_underscore_patch.sql | 0 ...90521000001_add_namespace_to_migration.sql | 25 ++++ ...0190521000002_migrate_migrations_table.sql | 25 ++++ .../src/modules/chain/migrations/index.js | 4 +- ...a.sql => 20160723182900_create_schema.sql} | 0 ...ws.sql => 20160723182901_create_views.sql} | 0 .../src/modules/network/migrations/index.js | 2 +- ...a.sql => 20160723182900_create_schema.sql} | 0 12 files changed, 166 insertions(+), 24 deletions(-) rename framework/src/controller/migrations/sql/{ => updates}/20160723182900_create_schema.sql (70%) rename framework/src/controller/migrations/sql/{ => updates}/20180205000000_underscore_patch.sql (100%) create mode 100644 framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql create mode 100644 framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql rename framework/src/modules/chain/migrations/sql/{20160723182901_create_schema.sql => 20160723182900_create_schema.sql} (100%) rename framework/src/modules/chain/migrations/sql/{20160723182903_create_views.sql => 20160723182901_create_views.sql} (100%) rename framework/src/modules/network/migrations/sql/{20160723182902_create_schema.sql => 20160723182900_create_schema.sql} (100%) diff --git a/framework/src/controller/controller.js b/framework/src/controller/controller.js index 5af42a63384..ee2877a3547 100644 --- a/framework/src/controller/controller.js +++ b/framework/src/controller/controller.js @@ -25,10 +25,7 @@ const { DuplicateAppInstanceError } = require('../errors'); const { validateModuleSpec } = require('./validator'); const ApplicationState = require('./application_state'); const { createStorageComponent } = require('../components/storage'); -const { - MigrationEntity, - migrations: frameworkMigrations, -} = require('./migrations'); +const { MigrationEntity } = require('./migrations'); const isPidRunning = async pid => psList().then(list => list.some(x => x.pid === pid)); @@ -196,16 +193,12 @@ class Controller { } async _loadMigrations(applicationMigrationsObj) { - const frameworkMigrationsObj = { - framework: frameworkMigrations, - }; - const storageConfig = this.config.components.storage; this.storage = createStorageComponent(storageConfig, this.logger); this.storage.registerEntity('Migration', MigrationEntity); await this.storage.bootstrap(); - await this.storage.entities.Migration.apply(frameworkMigrationsObj); - await this.storage.entities.Migration.apply(applicationMigrationsObj); + await this.storage.entities.Migration.applyInternal(); + await this.storage.entities.Migration.applyList(applicationMigrationsObj); return true; } diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration.js index 3c28cdf5503..cdb579b9701 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -38,6 +38,7 @@ const sqlFiles = { * @typedef {Object} Migration * @property {string} id * @property {string} name + * @property {string} namespace */ /** @@ -53,6 +54,11 @@ const sqlFiles = { * @property {string} [name_ne] * @property {string} [name_in] * @property {string} [name_like] + * @property {string} [namespace] + * @property {string} [namespace_eql] + * @property {string} [namespace_ne] + * @property {string} [namespace_in] + * @property {string} [namespace_like] */ class Migration extends BaseEntity { @@ -66,6 +72,7 @@ class Migration extends BaseEntity { this.addField('id', 'string', { filter: TEXT }); this.addField('name', 'string', { filter: TEXT }); + this.addField('namespace', 'string', { filter: TEXT }); const defaultSort = { sort: 'id:asc' }; this.extendDefaultOptions(defaultSort); @@ -217,7 +224,19 @@ class Migration extends BaseEntity { * * @returns {Promise} Promise object that resolves with a boolean. */ - async hasMigrations() { + async hasInternalMigrationsTable() { + const hasMigrations = await this.adapter.execute( + "SELECT table_name hasMigrations FROM information_schema.tables WHERE table_name = 'internal_migrations';" + ); + return !!hasMigrations.length; + } + + /** + * Verifies presence of the 'migrations' OID named relation. + * + * @returns {Promise} Promise object that resolves with a boolean. + */ + async hasMigrationsTable() { const hasMigrations = await this.adapter.execute( "SELECT table_name hasMigrations FROM information_schema.tables WHERE table_name = 'migrations';" ); @@ -230,9 +249,47 @@ class Migration extends BaseEntity { * @returns {Promise} * Promise object that resolves with either 0 or id of the last migration record. */ - async getLastId() { - const result = await this.get({}, { sort: 'id:DESC', limit: 1 }); - return result.length ? parseInt(result[0].id) : null; + async getLastInternalMigrationId() { + const lastID = await this.adapter.execute( + 'SELECT id FROM internal_migrations ORDER BY id DESC LIMIT 1;' + ); + + return lastID ? lastID[0].id : 0; + } + + /** + * Reads 'sql/migrations/updates' folder and returns an array of objects for further processing. + * + * @param {number} lastMigrationId + * @returns {Promise>} + * Promise object that resolves with an array of objects `{id, name, path, file}`. + */ + readPendingInternalMigrations(lastMigrationId) { + const updatesPath = path.join(__dirname, './sql/updates'); + return fs.readdir(updatesPath).then(files => + files + .map(migrationFile => { + const migration = migrationFile.match(/(\d+)_(.+).sql/); + return ( + migration && { + id: migration[1], + name: migration[2], + path: `${updatesPath}/${migrationFile}`, + } + ); + }) + .sort((a, b) => a.id - b.id) // Sort by migration ID, ascending + .filter( + migration => + migration && + fs.statSync(migration.path).isFile() && + (!lastMigrationId || +migration.id > lastMigrationId) + ) + .map(migration => { + migration.file = this.adapter.loadSQLFile(migration.path, ''); + return migration; + }) + ); } /** @@ -272,10 +329,23 @@ class Migration extends BaseEntity { }, []); } + async applyPendingInternalMigration(pendingMigration, tx) { + await this.adapter.executeFile(pendingMigration.file, {}, {}, tx); + const query = this.adapter.parseQueryComponent( + 'INSERT INTO "internal_migrations" ("id", "name") VALUES (${id}, ${name});', + { id: pendingMigration.id, name: pendingMigration.name } + ); + await this.adapter.execute(query, {}, {}, tx); + } + async applyPendingMigration(pendingMigration, tx) { await this.adapter.executeFile(pendingMigration.file, {}, {}, tx); await this.create( - { id: pendingMigration.id, name: pendingMigration.name }, + { + id: pendingMigration.id, + name: pendingMigration.name, + namespace: pendingMigration.namespace, + }, {}, tx ); @@ -287,11 +357,35 @@ class Migration extends BaseEntity { * * @returns {Promise} Promise object that resolves with `undefined`. */ - async apply(migrationsObj) { - const hasMigrations = await this.hasMigrations(); - const executedMigrationIDs = hasMigrations + async applyInternal() { + const hasInternalMigrationsTable = await this.hasInternalMigrationsTable(); + const lastId = hasInternalMigrationsTable + ? await this.getLastInternalMigrationId() + : 0; + const pendingMigrations = await this.readPendingInternalMigrations(lastId); + + if (pendingMigrations.length > 0) { + // eslint-disable-next-line no-restricted-syntax + for (const migration of pendingMigrations) { + const execute = tx => this.applyPendingInternalMigration(migration, tx); + // eslint-disable-next-line no-await-in-loop + await this.begin('migrations:applyInternal', execute); + } + } + } + + /** + * Applies a cumulative update: all pending migrations + runtime. + * Each update+insert execute within their own SAVEPOINT, to ensure data integrity on the updates level. + * + * @returns {Promise} Promise object that resolves with `undefined`. + */ + async applyList(migrationsObj) { + const hasMigrationsTable = await this.hasMigrationsTable(); + const executedMigrationIDs = hasMigrationsTable ? (await this.get({}, { limit: null })).map(m => m.id) : []; + const pendingMigrations = await this.readPending( migrationsObj, executedMigrationIDs diff --git a/framework/src/controller/migrations/sql/get.sql b/framework/src/controller/migrations/sql/get.sql index 4872370943b..6a9a1bf3e0b 100644 --- a/framework/src/controller/migrations/sql/get.sql +++ b/framework/src/controller/migrations/sql/get.sql @@ -13,8 +13,7 @@ */ SELECT - "id", - "name" + * FROM migrations diff --git a/framework/src/controller/migrations/sql/20160723182900_create_schema.sql b/framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql similarity index 70% rename from framework/src/controller/migrations/sql/20160723182900_create_schema.sql rename to framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql index 7bb943bdb8e..95960417eda 100644 --- a/framework/src/controller/migrations/sql/20160723182900_create_schema.sql +++ b/framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql @@ -14,12 +14,18 @@ /* - DESCRIPTION: Creates schema. + DESCRIPTION: Creates `internal_migrations` table to manage framework migrations + and `migrations` table to manage app/modules migrations. PARAMETERS: None */ /* Tables */ +CREATE TABLE IF NOT EXISTS "internal_migrations"( + "id" VARCHAR(22) NOT NULL PRIMARY KEY, + "name" TEXT NOT NULL +); + CREATE TABLE IF NOT EXISTS "migrations"( "id" VARCHAR(22) NOT NULL PRIMARY KEY, "name" TEXT NOT NULL diff --git a/framework/src/controller/migrations/sql/20180205000000_underscore_patch.sql b/framework/src/controller/migrations/sql/updates/20180205000000_underscore_patch.sql similarity index 100% rename from framework/src/controller/migrations/sql/20180205000000_underscore_patch.sql rename to framework/src/controller/migrations/sql/updates/20180205000000_underscore_patch.sql diff --git a/framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql b/framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql new file mode 100644 index 00000000000..ef2604a6331 --- /dev/null +++ b/framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql @@ -0,0 +1,25 @@ +/* + * Copyright © 2018 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + */ + + +/* + DESCRIPTION: Add `namespace` column to migrations table + and replace primary key by a composite key using id and namespace + + PARAMETERS: None +*/ + +ALTER TABLE "migrations" ADD COLUMN IF NOT EXISTS "namespace" TEXT NOT NULL DEFAULT 'undefined'; +ALTER TABLE "migrations" DROP CONSTRAINT "migrations_pkey"; +ALTER TABLE "migrations" ADD CONSTRAINT "migrations_pkey" PRIMARY KEY ("id", "namespace"); diff --git a/framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql b/framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql new file mode 100644 index 00000000000..5f1fe3225d7 --- /dev/null +++ b/framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql @@ -0,0 +1,25 @@ +/* + * Copyright © 2018 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + */ + + +/* + DESCRIPTION: Remove migrations-related entries and migrate existing entries namespace + + PARAMETERS: None +*/ + +DELETE FROM "migrations" WHERE id = '20180205000000'; +INSERT INTO "migrations" ("id", "name", "namespace") VALUES ('20160723182900', 'create_schema', 'network'); +UPDATE "migrations" SET namespace = 'network' WHERE id IN ('20161016133824', '20170113181857', '20171207000001', '20171227155620', '20180205000002', '20180327170000', '20181106000006', '20190103000001', '20190111111557'); +UPDATE "migrations" SET namespace = 'chain' WHERE namespace = 'undefined'; diff --git a/framework/src/modules/chain/migrations/index.js b/framework/src/modules/chain/migrations/index.js index 3dad1fdea51..b6ff6e9f81f 100644 --- a/framework/src/modules/chain/migrations/index.js +++ b/framework/src/modules/chain/migrations/index.js @@ -17,8 +17,8 @@ const path = require('path'); const migrations = [ - path.join(path.dirname(__filename), './sql/20160723182901_create_schema.sql'), - path.join(path.dirname(__filename), './sql/20160723182903_create_views.sql'), + path.join(path.dirname(__filename), './sql/20160723182900_create_schema.sql'), + path.join(path.dirname(__filename), './sql/20160723182901_create_views.sql'), path.join( path.dirname(__filename), './sql/20160724114255_create_memory_tables.sql' diff --git a/framework/src/modules/chain/migrations/sql/20160723182901_create_schema.sql b/framework/src/modules/chain/migrations/sql/20160723182900_create_schema.sql similarity index 100% rename from framework/src/modules/chain/migrations/sql/20160723182901_create_schema.sql rename to framework/src/modules/chain/migrations/sql/20160723182900_create_schema.sql diff --git a/framework/src/modules/chain/migrations/sql/20160723182903_create_views.sql b/framework/src/modules/chain/migrations/sql/20160723182901_create_views.sql similarity index 100% rename from framework/src/modules/chain/migrations/sql/20160723182903_create_views.sql rename to framework/src/modules/chain/migrations/sql/20160723182901_create_views.sql diff --git a/framework/src/modules/network/migrations/index.js b/framework/src/modules/network/migrations/index.js index 999b2a958a8..2bb9810a3f2 100644 --- a/framework/src/modules/network/migrations/index.js +++ b/framework/src/modules/network/migrations/index.js @@ -17,7 +17,7 @@ const path = require('path'); const migrations = [ - path.join(path.dirname(__filename), './sql/20160723182902_create_schema.sql'), + path.join(path.dirname(__filename), './sql/20160723182900_create_schema.sql'), path.join( path.dirname(__filename), './sql/20161016133824_add_broadhash_column_to_peers.sql' diff --git a/framework/src/modules/network/migrations/sql/20160723182902_create_schema.sql b/framework/src/modules/network/migrations/sql/20160723182900_create_schema.sql similarity index 100% rename from framework/src/modules/network/migrations/sql/20160723182902_create_schema.sql rename to framework/src/modules/network/migrations/sql/20160723182900_create_schema.sql From 5f286e90b3fcf2f5ca967bca388b789d8aa2e5c3 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Tue, 21 May 2019 17:15:11 +0200 Subject: [PATCH 06/22] Filter pending migrations based on id and namespace --- .../src/controller/migrations/migration.js | 17 +++++++++-------- .../20190521000002_migrate_migrations_table.sql | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration.js index cdb579b9701..732846deca0 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -299,7 +299,7 @@ class Migration extends BaseEntity { * @returns {Promise>} * Promise object that resolves with an array of objects `{id, name, path, file}`. */ - async readPending(migrationsObj, executedMigrationIDs) { + async readPending(migrationsObj, savedMigrations) { // const updatesPath = `${this.sqlDirectory}/updates`; return Object.keys(migrationsObj).reduce((prev, namespace) => { const curr = migrationsObj[namespace] @@ -314,13 +314,17 @@ class Migration extends BaseEntity { } ); }) - .sort((a, b) => a.id - b.id) // Sort by migration ID, ascending .filter( migration => migration && fs.statSync(migration.path).isFile() && - !executedMigrationIDs.includes(migration.id) + !savedMigrations.find( + saved => + saved.id === migration.id && + saved.namespace === migration.namespace + ) ) + .sort((a, b) => a.id - b.id) // Sort by migration ID, ascending .map(migration => { migration.file = this.adapter.loadSQLFile(migration.path, ''); return migration; @@ -381,14 +385,11 @@ class Migration extends BaseEntity { * @returns {Promise} Promise object that resolves with `undefined`. */ async applyList(migrationsObj) { - const hasMigrationsTable = await this.hasMigrationsTable(); - const executedMigrationIDs = hasMigrationsTable - ? (await this.get({}, { limit: null })).map(m => m.id) - : []; + const savedMigrations = await this.get({}, { limit: null }); const pendingMigrations = await this.readPending( migrationsObj, - executedMigrationIDs + savedMigrations ); if (pendingMigrations.length > 0) { diff --git a/framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql b/framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql index 5f1fe3225d7..f4bbf49eaf4 100644 --- a/framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql +++ b/framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql @@ -19,7 +19,17 @@ PARAMETERS: None */ -DELETE FROM "migrations" WHERE id = '20180205000000'; -INSERT INTO "migrations" ("id", "name", "namespace") VALUES ('20160723182900', 'create_schema', 'network'); +-- Remove framework-related migration (20180205000000_underscore_patch.sql) +DELETE FROM "migrations" WHERE id = '20180205000000' AND name = 'underscore_patch'; + +-- Populate `namespace` with correct initial values when exist UPDATE "migrations" SET namespace = 'network' WHERE id IN ('20161016133824', '20170113181857', '20171207000001', '20171227155620', '20180205000002', '20180327170000', '20181106000006', '20190103000001', '20190111111557'); UPDATE "migrations" SET namespace = 'chain' WHERE namespace = 'undefined'; + +-- Duplicate 20160723182900_create_schema.sql migration for `network` module if migration exists for `chain` module +-- That is required because 20160723182900_create_schema.sql was splited into two different files (one for each module) +INSERT INTO "migrations" ("id", "name", "namespace") +SELECT '20160723182900', 'create_schema', 'network' +WHERE EXISTS ( + SELECT * FROM "migrations" WHERE id = '20160723182900' AND name = 'create_schema' AND namespace = 'chain' +); From 577b4c36e0815ca4c726ca230587089c5c046892 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Tue, 21 May 2019 18:11:52 +0200 Subject: [PATCH 07/22] Apply migration changes to storageSandbox --- .../src/controller/migrations/migration.js | 2 +- framework/test/mocha/common/application.js | 12 +++++++++++- framework/test/mocha/common/storage_sandbox.js | 17 +++++++++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration.js index 732846deca0..2355dcc38a5 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -397,7 +397,7 @@ class Migration extends BaseEntity { for (const migration of pendingMigrations) { const execute = tx => this.applyPendingMigration(migration, tx); // eslint-disable-next-line no-await-in-loop - await this.begin('migrations:applyAll', execute); + await this.begin('migrations:applyList', execute); } } } diff --git a/framework/test/mocha/common/application.js b/framework/test/mocha/common/application.js index 771532c4094..a370ab693b5 100644 --- a/framework/test/mocha/common/application.js +++ b/framework/test/mocha/common/application.js @@ -41,6 +41,15 @@ const modulesInit = { '../../../src/modules/chain/submodules/process_transactions.js', }; +const ChainModule = require('../../../src/modules/chain'); +const NetworkModule = require('../../../src/modules/network'); +const HttpAPIModule = require('../../../src/modules/http_api'); + +const modulesMigrations = {}; +modulesMigrations[ChainModule.alias] = ChainModule.migrations; +modulesMigrations[NetworkModule.alias] = NetworkModule.migrations; +modulesMigrations[HttpAPIModule.alias] = HttpAPIModule.migrations; + function init(options, cb) { options = options || {}; options.scope = options.scope ? options.scope : {}; @@ -111,7 +120,8 @@ async function __init(sandbox, initScope) { }) .then(async status => { if (status) { - await storage.entities.Migration.applyAll(); + await storage.entities.Migration.applyInternal(); + await storage.entities.Migration.applyList(modulesMigrations); } }); diff --git a/framework/test/mocha/common/storage_sandbox.js b/framework/test/mocha/common/storage_sandbox.js index 6cf05513650..f3a2579f24b 100644 --- a/framework/test/mocha/common/storage_sandbox.js +++ b/framework/test/mocha/common/storage_sandbox.js @@ -25,7 +25,6 @@ const { const { Account, Block, - Migration, Round, Transaction, } = require('../../../src/modules/chain/components/storage/entities'); @@ -34,6 +33,19 @@ const { Peer, } = require('../../../src/modules/network/components/storage/entities'); +const { + MigrationEntity: Migration, +} = require('../../../src/controller/migrations'); + +const ChainModule = require('../../../src/modules/chain'); +const NetworkModule = require('../../../src/modules/network'); +const HttpAPIModule = require('../../../src/modules/http_api'); + +const modulesMigrations = {}; +modulesMigrations[ChainModule.alias] = ChainModule.migrations; +modulesMigrations[NetworkModule.alias] = NetworkModule.migrations; +modulesMigrations[HttpAPIModule.alias] = HttpAPIModule.migrations; + const dbNames = []; /** @@ -124,7 +136,8 @@ class StorageSandbox extends Storage { async _createSchema() { try { - await this.entities.Migration.applyAll(); + await this.entities.Migration.applyInternal(); + await this.entities.Migration.applyList(modulesMigrations); } catch (err) { Promise.reject(err); } From 41219d7fba0db9cd94e9671096d823f200390d7e Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Wed, 22 May 2019 11:05:07 +0200 Subject: [PATCH 08/22] Fix migration entity unit test --- .../components/storage/entities/migration.js | 317 ++++++++++++++---- 1 file changed, 247 insertions(+), 70 deletions(-) diff --git a/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js b/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js index 3fd5bbd9731..00633deece4 100644 --- a/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js +++ b/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js @@ -26,10 +26,14 @@ const { }, } = require('../../../../../../../../src/components/storage'); const { - Migration, -} = require('../../../../../../../../src/modules/chain/components/storage/entities'); + MigrationEntity, +} = require('../../../../../../../../src/controller/migrations'); const storageSandbox = require('../../../../../../common/storage_sandbox'); +const ChainModule = require('../../../../../../../../src/modules/chain'); +const NetworkModule = require('../../../../../../../../src/modules/network'); +const HttpAPIModule = require('../../../../../../../../src/modules/http_api'); + describe('Migration', () => { let adapter; let validMigrationFields; @@ -66,6 +70,11 @@ describe('Migration', () => { 'name_ne', 'name_in', 'name_like', + 'namespace', + 'namespace_eql', + 'namespace_ne', + 'namespace_in', + 'namespace_like', ]; invalidFilter = { @@ -98,7 +107,7 @@ describe('Migration', () => { adapter = storage.adapter; - addFieldSpy = sinonSandbox.spy(Migration.prototype, 'addField'); + addFieldSpy = sinonSandbox.spy(MigrationEntity.prototype, 'addField'); }); afterEach(async () => { @@ -106,23 +115,23 @@ describe('Migration', () => { }); it('should be a constructable function', async () => { - expect(Migration.prototype.constructor).not.to.be.null; - expect(Migration.prototype.constructor.name).to.be.eql('Migration'); + expect(MigrationEntity.prototype.constructor).not.to.be.null; + expect(MigrationEntity.prototype.constructor.name).to.be.eql('Migration'); }); it('should extend BaseEntity', async () => { - expect(Migration.prototype instanceof BaseEntity).to.be.true; + expect(MigrationEntity.prototype instanceof BaseEntity).to.be.true; }); describe('constructor()', () => { it('should accept only one mandatory parameter', async () => { - expect(Migration.prototype.constructor.length).to.be.eql(1); + expect(MigrationEntity.prototype.constructor.length).to.be.eql(1); }); it('should have called super', async () => { // The reasoning here is that if the parent's contstructor was called // the properties from the parent are present in the extending object - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(typeof migration.parseFilters).to.be.eql('function'); expect(typeof migration.addFilter).to.be.eql('function'); expect(typeof migration.addField).to.be.eql('function'); @@ -135,19 +144,19 @@ describe('Migration', () => { }); it('should assign proper sql', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(migration.SQLs).to.include.all.keys(validMigrationSQLs); }); it('should call addField the exact number of times', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(addFieldSpy.callCount).to.eql( Object.keys(migration.fields).length ); }); it('should setup correct fields', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(migration.fields).to.include.all.keys(validMigrationFields); }); @@ -156,7 +165,7 @@ describe('Migration', () => { describe('getOne()', () => { it('should call _getResults with the correct expectedResultCount', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); const _getResultsStub = sinonSandbox .stub(migration, '_getResults') .returns(validMigration); @@ -168,7 +177,7 @@ describe('Migration', () => { describe('get()', () => { it('should call _getResults with the correct expectedResultCount', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); const _getResultsStub = sinonSandbox .stub(migration, '_getResults') .returns(validMigration); @@ -180,28 +189,28 @@ describe('Migration', () => { describe('_getResults()', () => { it('should accept only valid filters', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(() => { migration.getOne(validFilter); }).not.to.throw(NonSupportedFilterTypeError); }); it('should throw error for invalid filters', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(() => { migration.getOne(invalidFilter); }).to.throw(NonSupportedFilterTypeError); }); it('should accept only valid options', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(() => { migration.getOne(validFilter, validOptions); }).not.to.throw(NonSupportedOptionError); }); it('should throw error for invalid options', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(() => { migration.getOne(validFilter, invalidOptions); }).to.throw(NonSupportedOptionError); @@ -209,7 +218,7 @@ describe('Migration', () => { it('should accept "tx" as last parameter and pass to adapter.executeFile', async () => { // Arrange - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); const getSpy = sinonSandbox.spy(migration, 'get'); // Act & Assert await migration.begin('testTX', async tx => { @@ -225,7 +234,7 @@ describe('Migration', () => { describe('filters', () => { // To make add/remove filters we add their tests. it('should have only specific filters', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(migration.getFilters()).to.eql(validFilters); }); // For each filter type @@ -235,13 +244,17 @@ describe('Migration', () => { describe('update()', () => { it('should always throw NonSupportedOperationError', async () => { - expect(Migration.prototype.update).to.throw(NonSupportedOperationError); + expect(MigrationEntity.prototype.update).to.throw( + NonSupportedOperationError + ); }); }); describe('delete()', () => { it('should always throw NonSupportedOperationError', async () => { - expect(Migration.prototype.delete).to.throw(NonSupportedOperationError); + expect(MigrationEntity.prototype.delete).to.throw( + NonSupportedOperationError + ); }); }); @@ -259,21 +272,21 @@ describe('Migration', () => { }); it('should accept only valid filters', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(() => { migration.isPersisted(validFilter); }).not.to.throw(NonSupportedFilterTypeError); }); it('should throw error for invalid filters', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); expect(() => { migration.isPersisted(invalidFilter); }).to.throw(NonSupportedFilterTypeError); }); it('should call mergeFilters with proper params', async () => { - const migration = new Migration(localAdapter); + const migration = new MigrationEntity(localAdapter); migration.mergeFilters = sinonSandbox.stub(); migration.parseFilters = sinonSandbox.stub(); migration.isPersisted(validFilter); @@ -281,7 +294,7 @@ describe('Migration', () => { }); it('should call parseFilters with proper params', async () => { - const migration = new Migration(localAdapter); + const migration = new MigrationEntity(localAdapter); migration.mergeFilters = sinonSandbox.stub().returns(validFilter); migration.parseFilters = sinonSandbox.stub(); migration.isPersisted(validFilter); @@ -289,7 +302,7 @@ describe('Migration', () => { }); it('should call adapter.executeFile with proper params', async () => { - const migration = new Migration(localAdapter); + const migration = new MigrationEntity(localAdapter); migration.mergeFilters = sinonSandbox.stub().returns(validFilter); migration.parseFilters = sinonSandbox.stub(); migration.getUpdateSet = sinonSandbox.stub(); @@ -328,7 +341,7 @@ describe('Migration', () => { describe('mergeFilters()', () => { it('should accept filters as single object', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); const mergeFiltersSpy = sinonSandbox.spy(migration, 'mergeFilters'); expect(() => { migration.get(validFilter); @@ -337,7 +350,7 @@ describe('Migration', () => { }); it('should accept filters as array of objects', async () => { - const migration = new Migration(adapter); + const migration = new MigrationEntity(adapter); const mergeFiltersSpy = sinonSandbox.spy(migration, 'mergeFilters'); expect(() => { migration.get([validFilter, validFilter]); @@ -351,26 +364,73 @@ describe('Migration', () => { }); describe('Schema Updates methods', () => { - let files; - let fileIds; + let internalMigrationIDs; + let savedMigrations; + + const modulesMigrations = {}; + modulesMigrations[ChainModule.alias] = ChainModule.migrations; + modulesMigrations[NetworkModule.alias] = NetworkModule.migrations; + modulesMigrations[HttpAPIModule.alias] = HttpAPIModule.migrations; before(async () => { - files = await fs.readdir( + const internalMigrationFiles = await fs.readdir( path.join( __dirname, - '../../../../../../../../src/modules/chain/components/storage/sql/migrations/updates' + '../../../../../../../../src/controller/migrations/sql/updates' ) ); - fileIds = files.map(f => f.match(/(\d+)_(.+).sql/)[1]).sort(); + internalMigrationIDs = internalMigrationFiles + .map(f => f.match(/(\d+)_(.+).sql/)[1]) + .sort(); + + savedMigrations = Object.keys(modulesMigrations).reduce( + (prev, namespace) => { + const curr = modulesMigrations[namespace].map(migrationFile => { + const migration = migrationFile.match(/(\d+)_(.+).sql/); + return ( + migration && { + id: migration[1], + name: migration[2], + namespace, + } + ); + }); + return prev.concat(curr); + }, + [] + ); }); afterEach(async () => { sinonSandbox.restore(); }); - describe('hasMigrations()', () => { + describe('hasInternalMigrationsTable()', () => { + it('should resolve with true if internal_migrations table exists', async () => { + expect(await storage.entities.Migration.hasInternalMigrationsTable()).to + .be.true; + }); + + it('should resolve with false if migrations table does not exists', async () => { + // Create backup table and drop migrations table + await storage.entities.Migration.adapter.execute( + 'CREATE TABLE temp_migrations AS TABLE internal_migrations; DROP TABLE internal_migrations;' + ); + + expect(await storage.entities.Migration.hasInternalMigrationsTable()).to + .be.false; + + // Restore the migrations table and drop the backup + await storage.entities.Migration.adapter.execute( + 'CREATE TABLE internal_migrations AS TABLE temp_migrations; DROP TABLE temp_migrations;' + ); + }); + }); + + describe('hasMigrationsTable()', () => { it('should resolve with true if migrations table exists', async () => { - expect(await storage.entities.Migration.hasMigrations()).to.be.true; + expect(await storage.entities.Migration.hasMigrationsTable()).to.be + .true; }); it('should resolve with false if migrations table does not exists', async () => { @@ -379,7 +439,8 @@ describe('Migration', () => { 'CREATE TABLE temp_migrations AS TABLE migrations; DROP TABLE migrations;' ); - expect(await storage.entities.Migration.hasMigrations()).to.be.false; + expect(await storage.entities.Migration.hasMigrationsTable()).to.be + .false; // Restore the migrations table and drop the backup await storage.entities.Migration.adapter.execute( @@ -388,43 +449,45 @@ describe('Migration', () => { }); }); - describe('getLastId', () => { + describe('getLastInternalMigrationId', () => { it('should use the correct filters for get', async () => { - sinonSandbox.spy(storage.entities.Migration, 'get'); - await storage.entities.Migration.getLastId(); - expect(storage.entities.Migration.get.firstCall.args[0]).to.be.eql({}); - expect(storage.entities.Migration.get.firstCall.args[1]).to.be.eql({ - sort: 'id:DESC', - limit: 1, - }); + sinonSandbox.spy(storage.entities.Migration.adapter, 'execute'); + await storage.entities.Migration.getLastInternalMigrationId(); + expect( + storage.entities.Migration.adapter.execute.firstCall.args[0] + ).to.be.eql( + 'SELECT id FROM internal_migrations ORDER BY id DESC LIMIT 1;' + ); }); it('should return id of the last migration file', async () => { - const result = await storage.entities.Migration.getLastId(); - expect(result).to.be.eql(parseInt(fileIds[fileIds.length - 1])); + const result = await storage.entities.Migration.getLastInternalMigrationId(); + expect(result).to.be.eql( + internalMigrationIDs[internalMigrationIDs.length - 1] + ); }); }); - describe('readPending', () => { + describe('readPendingInternalMigrations', () => { it('should resolve with list of pending files if there exists any', async () => { - const pending = (await storage.entities.Migration.readPending( - fileIds[0] + const pending = (await storage.entities.Migration.readPendingInternalMigrations( + internalMigrationIDs[0] )).map(f => f.id); - fileIds.splice(0, 1); - expect(pending).to.be.eql(fileIds); + internalMigrationIDs.splice(0, 1); + expect(pending).to.be.eql(internalMigrationIDs); }); it('should resolve with empty array if there is no pending migration', async () => { - const pending = await storage.entities.Migration.readPending( - fileIds[fileIds.length - 1] + const pending = await storage.entities.Migration.readPendingInternalMigrations( + internalMigrationIDs[internalMigrationIDs.length - 1] ); return expect(pending).to.be.empty; }); it('should resolve with the list in correct format', async () => { - const pending = await storage.entities.Migration.readPending( - fileIds[0] + const pending = await storage.entities.Migration.readPendingInternalMigrations( + internalMigrationIDs[0] ); expect(pending).to.be.an('array'); @@ -435,28 +498,142 @@ describe('Migration', () => { }); }); - describe('applyAll()', () => { + describe('readPending', () => { + it('should resolve with list of pending files if there exists any', async () => { + const pendingMigrationsMock = savedMigrations.slice(0, 2); + const savedMigrationsMock = savedMigrations.slice(2); + const pendingMigrations = await storage.entities.Migration.readPending( + modulesMigrations, + savedMigrationsMock + ); + expect( + pendingMigrations.map(m => ({ + id: m.id, + name: m.name, + namespace: m.namespace, + })) + ).to.be.eql(pendingMigrationsMock); + }); + + it('should resolve with empty array if there is no pending migration', async () => { + const pending = await storage.entities.Migration.readPending( + modulesMigrations, + savedMigrations + ); + + return expect(pending).to.be.empty; + }); + + it('should resolve with the list in correct format', async () => { + const savedMigrationsMock = savedMigrations.slice(2); + const pendingMigrations = await storage.entities.Migration.readPending( + modulesMigrations, + savedMigrationsMock + ); + + expect(pendingMigrations).to.be.an('array'); + expect(pendingMigrations[0]).to.have.all.keys( + 'id', + 'name', + 'namespace', + 'path', + 'file' + ); + expect(pendingMigrations[0].file).to.be.instanceOf( + storage.entities.Migration.adapter.pgp.QueryFile + ); + }); + }); + + describe('applyInternal()', () => { let updates; beforeEach(async () => { - updates = await storage.entities.Migration.readPending(fileIds[0]); + updates = await storage.entities.Migration.readPendingInternalMigrations( + internalMigrationIDs[0] + ); }); it('should call hasMigrations()', async () => { - sinonSandbox.spy(storage.entities.Migration, 'hasMigrations'); - await storage.entities.Migration.applyAll(); - expect(storage.entities.Migration.hasMigrations).to.be.calledOnce; + sinonSandbox.spy( + storage.entities.Migration, + 'hasInternalMigrationsTable' + ); + await storage.entities.Migration.applyInternal(); + expect(storage.entities.Migration.hasInternalMigrationsTable).to.be + .calledOnce; + }); + + it('should call getLastInternalMigrationId()', async () => { + sinonSandbox.spy( + storage.entities.Migration, + 'getLastInternalMigrationId' + ); + await storage.entities.Migration.applyInternal(); + expect(storage.entities.Migration.getLastInternalMigrationId).to.be + .calledOnce; + }); + + it('should call readPendingInternalMigrations()', async () => { + sinonSandbox.spy( + storage.entities.Migration, + 'readPendingInternalMigrations' + ); + await storage.entities.Migration.applyInternal(); + expect(storage.entities.Migration.readPendingInternalMigrations).to.be + .calledOnce; + }); + + it('should apply all pending internal migrations in independent transactions', async () => { + sinonSandbox.spy(storage.entities.Migration, 'begin'); + sinonSandbox + .stub(storage.entities.Migration, 'readPendingInternalMigrations') + .resolves(updates); + sinonSandbox + .stub(storage.entities.Migration, 'applyPendingInternalMigration') + .resolves(null); + + await storage.entities.Migration.applyInternal(); + + expect(storage.entities.Migration.begin.callCount).to.be.eql( + updates.length + ); + expect( + storage.entities.Migration.begin + .getCalls() + .filter(aCall => aCall.args[0] !== 'migrations:applyInternal') + .length + ).to.be.eql(0); + + const applyPendingInternalMigrationCalls = storage.entities.Migration.applyPendingInternalMigration.getCalls(); + + applyPendingInternalMigrationCalls.forEach((aCall, idx) => { + expect(aCall.args[0]).to.be.eql(updates[idx]); + expect(aCall.args[1].constructor.name).to.be.eql('Task'); + }); }); + }); + + describe('applyList()', () => { + let pendingMigrations; - it('should call getLastId()', async () => { - sinonSandbox.spy(storage.entities.Migration, 'getLastId'); - await storage.entities.Migration.applyAll(); - expect(storage.entities.Migration.getLastId).to.be.calledOnce; + beforeEach(async () => { + const savedMigrationsMock = savedMigrations.slice(2); + pendingMigrations = await storage.entities.Migration.readPending( + modulesMigrations, + savedMigrationsMock + ); + }); + + it('should call this.get()', async () => { + sinonSandbox.spy(storage.entities.Migration, 'get'); + await storage.entities.Migration.applyList(modulesMigrations); + expect(storage.entities.Migration.get).to.be.calledOnce; }); it('should call readPending()', async () => { sinonSandbox.spy(storage.entities.Migration, 'readPending'); - await storage.entities.Migration.applyAll(); + await storage.entities.Migration.applyList(modulesMigrations); expect(storage.entities.Migration.readPending).to.be.calledOnce; }); @@ -464,26 +641,26 @@ describe('Migration', () => { sinonSandbox.spy(storage.entities.Migration, 'begin'); sinonSandbox .stub(storage.entities.Migration, 'readPending') - .resolves(updates); + .resolves(pendingMigrations); sinonSandbox .stub(storage.entities.Migration, 'applyPendingMigration') .resolves(null); - await storage.entities.Migration.applyAll(); + await storage.entities.Migration.applyList(modulesMigrations); expect(storage.entities.Migration.begin.callCount).to.be.eql( - updates.length + pendingMigrations.length ); expect( storage.entities.Migration.begin .getCalls() - .filter(aCall => aCall.args[0] !== 'migrations:applyAll').length + .filter(aCall => aCall.args[0] !== 'migrations:applyList').length ).to.be.eql(0); const applyPendingMigrationCalls = storage.entities.Migration.applyPendingMigration.getCalls(); applyPendingMigrationCalls.forEach((aCall, idx) => { - expect(aCall.args[0]).to.be.eql(updates[idx]); + expect(aCall.args[0]).to.be.eql(pendingMigrations[idx]); expect(aCall.args[1].constructor.name).to.be.eql('Task'); }); }); From f43412bdf759fcb9ab16758f1f8722b465c0fe89 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Wed, 22 May 2019 11:19:13 +0200 Subject: [PATCH 09/22] Removed unused code --- framework/src/controller/migrations/index.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/framework/src/controller/migrations/index.js b/framework/src/controller/migrations/index.js index 392bc3dc901..9f504df1462 100644 --- a/framework/src/controller/migrations/index.js +++ b/framework/src/controller/migrations/index.js @@ -14,17 +14,6 @@ 'use strict'; -const path = require('path'); - -const migrations = [ - path.join(path.dirname(__filename), './sql/20160723182900_create_schema.sql'), - path.join( - path.dirname(__filename), - './sql/20180205000000_underscore_patch.sql' - ), -]; - module.exports = { MigrationEntity: require('./migration'), - migrations, }; From bca24ee76de7e10e37f0cd3490314bb78a378b4a Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Wed, 22 May 2019 14:44:23 +0200 Subject: [PATCH 10/22] Code refactoring based on PR review --- framework/src/controller/controller.js | 3 +-- framework/src/controller/migrations/index.js | 4 +++- framework/src/controller/migrations/migration.js | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/framework/src/controller/controller.js b/framework/src/controller/controller.js index ee2877a3547..441f4b6decd 100644 --- a/framework/src/controller/controller.js +++ b/framework/src/controller/controller.js @@ -198,8 +198,7 @@ class Controller { this.storage.registerEntity('Migration', MigrationEntity); await this.storage.bootstrap(); await this.storage.entities.Migration.applyInternal(); - await this.storage.entities.Migration.applyList(applicationMigrationsObj); - return true; + return this.storage.entities.Migration.applyList(applicationMigrationsObj); } async _loadModules(modules, moduleOptions) { diff --git a/framework/src/controller/migrations/index.js b/framework/src/controller/migrations/index.js index 9f504df1462..bd7012fecb5 100644 --- a/framework/src/controller/migrations/index.js +++ b/framework/src/controller/migrations/index.js @@ -14,6 +14,8 @@ 'use strict'; +const MigrationEntity = require('./migration'); + module.exports = { - MigrationEntity: require('./migration'), + MigrationEntity, }; diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration.js index 2355dcc38a5..59d4a3f7ebc 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -250,11 +250,11 @@ class Migration extends BaseEntity { * Promise object that resolves with either 0 or id of the last migration record. */ async getLastInternalMigrationId() { - const lastID = await this.adapter.execute( + const [lastID] = await this.adapter.execute( 'SELECT id FROM internal_migrations ORDER BY id DESC LIMIT 1;' ); - return lastID ? lastID[0].id : 0; + return lastID ? lastID.id : 0; } /** @@ -274,7 +274,7 @@ class Migration extends BaseEntity { migration && { id: migration[1], name: migration[2], - path: `${updatesPath}/${migrationFile}`, + path: path.join(updatesPath, migrationFile), } ); }) From 702030353b460f2ee7036525fb6bb19319b1315a Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Wed, 22 May 2019 15:13:35 +0200 Subject: [PATCH 11/22] Filter migration list before sorting it --- framework/src/controller/migrations/migration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration.js index 59d4a3f7ebc..c3d9d58bd6d 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -278,13 +278,13 @@ class Migration extends BaseEntity { } ); }) - .sort((a, b) => a.id - b.id) // Sort by migration ID, ascending .filter( migration => migration && fs.statSync(migration.path).isFile() && (!lastMigrationId || +migration.id > lastMigrationId) ) + .sort((a, b) => a.id - b.id) // Sort by migration ID, ascending .map(migration => { migration.file = this.adapter.loadSQLFile(migration.path, ''); return migration; From dcc16a66befe98104745a8f543655268f24bce14 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Wed, 22 May 2019 15:29:49 +0200 Subject: [PATCH 12/22] Load all migration files from module migration folder --- .../src/modules/chain/migrations/index.js | 170 +----------------- .../src/modules/network/migrations/index.js | 44 +---- 2 files changed, 10 insertions(+), 204 deletions(-) diff --git a/framework/src/modules/chain/migrations/index.js b/framework/src/modules/chain/migrations/index.js index b6ff6e9f81f..8c7b92615c4 100644 --- a/framework/src/modules/chain/migrations/index.js +++ b/framework/src/modules/chain/migrations/index.js @@ -15,172 +15,12 @@ 'use strict'; const path = require('path'); +const fs = require('fs-extra'); -const migrations = [ - path.join(path.dirname(__filename), './sql/20160723182900_create_schema.sql'), - path.join(path.dirname(__filename), './sql/20160723182901_create_views.sql'), - path.join( - path.dirname(__filename), - './sql/20160724114255_create_memory_tables.sql' - ), - path.join( - path.dirname(__filename), - './sql/20160724132825_upcase_memory_table_addresses.sql' - ), - path.join( - path.dirname(__filename), - './sql/20160725173858_alter_mem_accounts_columns.sql' - ), - path.join( - path.dirname(__filename), - './sql/20160908120022_add_virgin_column_to_mem_accounts.sql' - ), - path.join( - path.dirname(__filename), - './sql/20160908215531_protect_mem_accounts_columns.sql' - ), - path.join( - path.dirname(__filename), - './sql/20161007153817_create_memory_table_indexes.sql' - ), - path.join( - path.dirname(__filename), - './sql/20170124071600_recreate_trs_list_view.sql' - ), - path.join( - path.dirname(__filename), - './sql/20170319001337_create_indexes.sql' - ), - path.join( - path.dirname(__filename), - './sql/20170321001337_create_rounds_fees_table.sql' - ), - path.join( - path.dirname(__filename), - './sql/20170328001337_recreate_trs_list_view.sql' - ), - path.join( - path.dirname(__filename), - './sql/20170403001337_calculate_blocks_rewards.sql' - ), - path.join(path.dirname(__filename), './sql/20170408001337_create_index.sql'), - path.join( - path.dirname(__filename), - './sql/20170422001337_recreate_calculate_blocks_rewards.sql' - ), - path.join( - path.dirname(__filename), - './sql/20170428001337_recreate_trs_lisk_view.sql' - ), - path.join( - path.dirname(__filename), - './sql/20170614155841_unique_delegates_constraint.sql' - ), - path.join( - path.dirname(__filename), - './sql/20170921105500_recreate_revert_mem_account_trigger.sql' - ), - path.join( - path.dirname(__filename), - './sql/20171207000006_create_transfer_trs_table.sql' - ), - path.join( - path.dirname(__filename), - './sql/20171207000007_recreate_full_block_list_view.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180205000001_drop_rewards_related_functions.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180205000003_create_rounds_rewards_table.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180205000004_apply_round_exception_mainnet.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180214164336_change_case_for_blocks_columns_in_mem_accounts.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180227120000_enforce_uppercase_trs_recipienId.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180419001337_alter_mem_blockid_column.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180423001337_remove_virgin_column.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180503114500_add_row_id_to_trs_list_view.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180814001337_add_indexes_for_trs_table.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180901001337_recreate_full_blocks_list_view.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180903001337_rename_rate_to_rank_in_delegates.sql' - ), - path.join( - path.dirname(__filename), - './sql/20181023001337_change_round_type_in_mem_rounds.sql' - ), - path.join( - path.dirname(__filename), - './sql/20181106000001_remove_duplicate_rows_and_add_unique_constraint_in_dapps.sql' - ), - path.join( - path.dirname(__filename), - './sql/20181106000002_remove_duplicate_rows_and_add_unique_constraint_in_votes.sql' - ), - path.join( - path.dirname(__filename), - './sql/20181106000003_remove_duplicate_rows_and_add_unique_constraint_in_transfer.sql' - ), - path.join( - path.dirname(__filename), - './sql/20181106000004_remove_duplicate_rows_and_add_unique_constraint_in_intransfer.sql' - ), - path.join( - path.dirname(__filename), - './sql/20181106000005_remove_duplicate_rows_and_add_unique_constraint_in_multisignatures.sql' - ), - path.join( - path.dirname(__filename), - './sql/20190104000001_add_recipient_public_key_to_full_block_list.sql' - ), - path.join( - path.dirname(__filename), - './sql/20190204170819_migrate_trs_dependant_tables_to_trs_trable.sql' - ), - path.join( - path.dirname(__filename), - './sql/20190313102300_drop_blocks_list_and_trs_list_views.sql' - ), - path.join( - path.dirname(__filename), - './sql/20190319111600_remove_unconfirmed_state.sql' - ), - path.join( - path.dirname(__filename), - './sql/20190319111700_remove_unconfirmed_state_from_mem_accounts_trigger.sql' - ), - path.join( - path.dirname(__filename), - './sql/20190410112400_add_asset_field_mem_accounts.sql' - ), -]; +const updatesPath = path.join(__dirname, './sql'); +const migrations = fs + .readdirSync(updatesPath) + .map(file => path.join(updatesPath, file)); module.exports = { migrations, diff --git a/framework/src/modules/network/migrations/index.js b/framework/src/modules/network/migrations/index.js index 2bb9810a3f2..8c7b92615c4 100644 --- a/framework/src/modules/network/migrations/index.js +++ b/framework/src/modules/network/migrations/index.js @@ -15,46 +15,12 @@ 'use strict'; const path = require('path'); +const fs = require('fs-extra'); -const migrations = [ - path.join(path.dirname(__filename), './sql/20160723182900_create_schema.sql'), - path.join( - path.dirname(__filename), - './sql/20161016133824_add_broadhash_column_to_peers.sql' - ), - path.join( - path.dirname(__filename), - './sql/20170113181857_add_constraints_to_peers.sql' - ), - path.join( - path.dirname(__filename), - './sql/20171207000001_remove_peers_dapp_table.sql' - ), - path.join( - path.dirname(__filename), - './sql/20171227155620_rename_port_to_ws_port.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180205000002_add_height_column_to_peers.sql' - ), - path.join( - path.dirname(__filename), - './sql/20180327170000_support_long_peer_version_numbers.sql' - ), - path.join( - path.dirname(__filename), - './sql/20181106000006_change_wsport_type_in_peers.sql' - ), - path.join( - path.dirname(__filename), - './sql/20190103000001_drop_peers_clock.sql' - ), - path.join( - path.dirname(__filename), - './sql/20190111111557_add_protocolVersion_column_to_peers.sql' - ), -]; +const updatesPath = path.join(__dirname, './sql'); +const migrations = fs + .readdirSync(updatesPath) + .map(file => path.join(updatesPath, file)); module.exports = { migrations, From f1cf62510ee721caa24d6dbfb14e8e439e94f34a Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Thu, 23 May 2019 10:37:59 +0200 Subject: [PATCH 13/22] Freeze migrations before set --- framework/src/controller/application.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/src/controller/application.js b/framework/src/controller/application.js index b4c68838a78..356cf09cbe2 100644 --- a/framework/src/controller/application.js +++ b/framework/src/controller/application.js @@ -263,13 +263,13 @@ class Application { * Register migrations with the application * * @param {Object} namespace - Migration namespace - * @param {Array} migrations - Migrations list. Format ['yyyyMMddHHmmss_name_of_migration.sql'] + * @param {Array} migrations - Migrations list. Format ['/path/to/migration/yyyyMMddHHmmss_name_of_migration.sql'] */ registerMigrations(namespace, migrations) { assert(namespace, 'Namespace is required'); assert(migrations instanceof Array, 'Migrations list should be an array'); const currentMigrations = this.getMigrations() || {}; - currentMigrations[namespace] = migrations; + currentMigrations[namespace] = Object.freeze(migrations); __private.migrations.set(this, currentMigrations); } From f6d277d993524d64bf3b84ae751104ecf5cfb230 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Thu, 23 May 2019 10:39:00 +0200 Subject: [PATCH 14/22] Move storage initialization to Controller constructor --- framework/src/controller/controller.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/framework/src/controller/controller.js b/framework/src/controller/controller.js index 441f4b6decd..16c9375c29b 100644 --- a/framework/src/controller/controller.js +++ b/framework/src/controller/controller.js @@ -71,6 +71,10 @@ class Controller { this.childrenList = []; this.channel = null; // Channel for controller this.bus = null; + + const storageConfig = config.components.storage; + this.storage = createStorageComponent(storageConfig, logger); + this.storage.registerEntity('Migration', MigrationEntity); } /** @@ -80,7 +84,7 @@ class Controller { * @param modules * @async */ - async load(modules, moduleOptions, migrations) { + async load(modules, moduleOptions, migrations = {}) { this.logger.info('Loading controller'); await this._setupDirectories(); await this._validatePidFile(); @@ -193,9 +197,6 @@ class Controller { } async _loadMigrations(applicationMigrationsObj) { - const storageConfig = this.config.components.storage; - this.storage = createStorageComponent(storageConfig, this.logger); - this.storage.registerEntity('Migration', MigrationEntity); await this.storage.bootstrap(); await this.storage.entities.Migration.applyInternal(); return this.storage.entities.Migration.applyList(applicationMigrationsObj); From 462fc2c9f548e84ebd9bbe9ce68160e7a803597c Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Thu, 23 May 2019 10:41:18 +0200 Subject: [PATCH 15/22] Remove unecessary alias on query --- framework/src/controller/migrations/migration.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration.js index c3d9d58bd6d..9ceb2708c21 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -225,10 +225,10 @@ class Migration extends BaseEntity { * @returns {Promise} Promise object that resolves with a boolean. */ async hasInternalMigrationsTable() { - const hasMigrations = await this.adapter.execute( - "SELECT table_name hasMigrations FROM information_schema.tables WHERE table_name = 'internal_migrations';" + const response = await this.adapter.execute( + "SELECT table_name FROM information_schema.tables WHERE table_name = 'internal_migrations';" ); - return !!hasMigrations.length; + return !!response.length; } /** @@ -237,10 +237,10 @@ class Migration extends BaseEntity { * @returns {Promise} Promise object that resolves with a boolean. */ async hasMigrationsTable() { - const hasMigrations = await this.adapter.execute( - "SELECT table_name hasMigrations FROM information_schema.tables WHERE table_name = 'migrations';" + const response = await this.adapter.execute( + "SELECT table_name FROM information_schema.tables WHERE table_name = 'migrations';" ); - return !!hasMigrations.length; + return !!response.length; } /** From 5a77e53a1c8539ff003e302eae7eb5aff7d2e7e9 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Thu, 23 May 2019 16:03:30 +0200 Subject: [PATCH 16/22] Prevent registering migrations to an existing namespace --- framework/src/controller/application.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/framework/src/controller/application.js b/framework/src/controller/application.js index 356cf09cbe2..bd97535ee89 100644 --- a/framework/src/controller/application.js +++ b/framework/src/controller/application.js @@ -153,6 +153,7 @@ class Application { __private.modules.set(this, {}); __private.transactions.set(this, {}); + __private.migrations.set(this, {}); const { TRANSACTION_TYPES } = constants; @@ -268,7 +269,12 @@ class Application { registerMigrations(namespace, migrations) { assert(namespace, 'Namespace is required'); assert(migrations instanceof Array, 'Migrations list should be an array'); - const currentMigrations = this.getMigrations() || {}; + assert( + !Object.keys(this.getMigrations()).includes(namespace), + `Migrations for "${namespace}" was already registered.` + ); + + const currentMigrations = this.getMigrations(); currentMigrations[namespace] = Object.freeze(migrations); __private.migrations.set(this, currentMigrations); } From 8574f536819de9cff185351b2c99621a25957b57 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Fri, 24 May 2019 15:20:19 +0200 Subject: [PATCH 17/22] Improve migration query --- .../sql/updates/20190521000001_add_namespace_to_migration.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql b/framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql index ef2604a6331..71e7a6a17e3 100644 --- a/framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql +++ b/framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql @@ -21,5 +21,5 @@ */ ALTER TABLE "migrations" ADD COLUMN IF NOT EXISTS "namespace" TEXT NOT NULL DEFAULT 'undefined'; -ALTER TABLE "migrations" DROP CONSTRAINT "migrations_pkey"; -ALTER TABLE "migrations" ADD CONSTRAINT "migrations_pkey" PRIMARY KEY ("id", "namespace"); +ALTER TABLE "migrations" DROP CONSTRAINT IF EXISTS "migrations_pkey"; +ALTER TABLE "migrations" ADD CONSTRAINT IF NOT EXISTS "migrations_pkey" PRIMARY KEY ("id", "namespace"); From ea9710976b3b47634df18b244340f7e3be01a34f Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Mon, 27 May 2019 16:09:10 +0200 Subject: [PATCH 18/22] Replace framework migrations for a static file Remove internal_migrations table as per review request --- framework/src/controller/controller.js | 6 +- .../src/controller/migrations/migration.js | 127 +++--------------- ...migrations_table.sql => define_schema.sql} | 21 ++- .../updates/20160723182900_create_schema.sql | 32 ----- .../20180205000000_underscore_patch.sql | 23 ---- ...90521000001_add_namespace_to_migration.sql | 25 ---- 6 files changed, 37 insertions(+), 197 deletions(-) rename framework/src/controller/migrations/sql/{updates/20190521000002_migrate_migrations_table.sql => define_schema.sql} (54%) delete mode 100644 framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql delete mode 100644 framework/src/controller/migrations/sql/updates/20180205000000_underscore_patch.sql delete mode 100644 framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql diff --git a/framework/src/controller/controller.js b/framework/src/controller/controller.js index 2e942dae74e..a6c3dd54f65 100644 --- a/framework/src/controller/controller.js +++ b/framework/src/controller/controller.js @@ -193,10 +193,10 @@ class Controller { } } - async _loadMigrations(applicationMigrationsObj) { + async _loadMigrations(migrationsObj) { await this.storage.bootstrap(); - await this.storage.entities.Migration.applyInternal(); - return this.storage.entities.Migration.applyList(applicationMigrationsObj); + await this.storage.entities.Migration.defineSchema(); + return this.storage.entities.Migration.applyAll(migrationsObj); } async _loadModules(modules, moduleOptions) { diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration.js index 9ceb2708c21..3bcc32df7d5 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -31,6 +31,7 @@ const sqlFiles = { select: 'get.sql', isPersisted: 'is_persisted.sql', create: 'create.sql', + defineSchema: 'define_schema.sql', }; /** @@ -220,87 +221,14 @@ class Migration extends BaseEntity { } /** - * Verifies presence of the 'migrations' OID named relation. + * Creates an array of objects with `{id, name, namespace, path}`, remove the ones already executed, sorts by ID ascending and add the file property * - * @returns {Promise} Promise object that resolves with a boolean. - */ - async hasInternalMigrationsTable() { - const response = await this.adapter.execute( - "SELECT table_name FROM information_schema.tables WHERE table_name = 'internal_migrations';" - ); - return !!response.length; - } - - /** - * Verifies presence of the 'migrations' OID named relation. - * - * @returns {Promise} Promise object that resolves with a boolean. - */ - async hasMigrationsTable() { - const response = await this.adapter.execute( - "SELECT table_name FROM information_schema.tables WHERE table_name = 'migrations';" - ); - return !!response.length; - } - - /** - * Gets id of the last migration record, or 0, if none exist. - * - * @returns {Promise} - * Promise object that resolves with either 0 or id of the last migration record. - */ - async getLastInternalMigrationId() { - const [lastID] = await this.adapter.execute( - 'SELECT id FROM internal_migrations ORDER BY id DESC LIMIT 1;' - ); - - return lastID ? lastID.id : 0; - } - - /** - * Reads 'sql/migrations/updates' folder and returns an array of objects for further processing. - * - * @param {number} lastMigrationId - * @returns {Promise>} - * Promise object that resolves with an array of objects `{id, name, path, file}`. - */ - readPendingInternalMigrations(lastMigrationId) { - const updatesPath = path.join(__dirname, './sql/updates'); - return fs.readdir(updatesPath).then(files => - files - .map(migrationFile => { - const migration = migrationFile.match(/(\d+)_(.+).sql/); - return ( - migration && { - id: migration[1], - name: migration[2], - path: path.join(updatesPath, migrationFile), - } - ); - }) - .filter( - migration => - migration && - fs.statSync(migration.path).isFile() && - (!lastMigrationId || +migration.id > lastMigrationId) - ) - .sort((a, b) => a.id - b.id) // Sort by migration ID, ascending - .map(migration => { - migration.file = this.adapter.loadSQLFile(migration.path, ''); - return migration; - }) - ); - } - - /** - * Reads/updates' folder and returns an array of objects for further processing. - * - * @param {number} lastMigrationId + * @param {Objec} migrationsObj - Object where the key is the migrations namespace and the value an array of migration's path + * @param {Array} savedMigrations - Array of objects with all migrations already executed before * @returns {Promise>} - * Promise object that resolves with an array of objects `{id, name, path, file}`. + * Promise object that resolves with an array of objects `{id, name, namespace, path, file}`. */ async readPending(migrationsObj, savedMigrations) { - // const updatesPath = `${this.sqlDirectory}/updates`; return Object.keys(migrationsObj).reduce((prev, namespace) => { const curr = migrationsObj[namespace] .map(migrationFile => { @@ -333,15 +261,6 @@ class Migration extends BaseEntity { }, []); } - async applyPendingInternalMigration(pendingMigration, tx) { - await this.adapter.executeFile(pendingMigration.file, {}, {}, tx); - const query = this.adapter.parseQueryComponent( - 'INSERT INTO "internal_migrations" ("id", "name") VALUES (${id}, ${name});', - { id: pendingMigration.id, name: pendingMigration.name } - ); - await this.adapter.execute(query, {}, {}, tx); - } - async applyPendingMigration(pendingMigration, tx) { await this.adapter.executeFile(pendingMigration.file, {}, {}, tx); await this.create( @@ -361,30 +280,7 @@ class Migration extends BaseEntity { * * @returns {Promise} Promise object that resolves with `undefined`. */ - async applyInternal() { - const hasInternalMigrationsTable = await this.hasInternalMigrationsTable(); - const lastId = hasInternalMigrationsTable - ? await this.getLastInternalMigrationId() - : 0; - const pendingMigrations = await this.readPendingInternalMigrations(lastId); - - if (pendingMigrations.length > 0) { - // eslint-disable-next-line no-restricted-syntax - for (const migration of pendingMigrations) { - const execute = tx => this.applyPendingInternalMigration(migration, tx); - // eslint-disable-next-line no-await-in-loop - await this.begin('migrations:applyInternal', execute); - } - } - } - - /** - * Applies a cumulative update: all pending migrations + runtime. - * Each update+insert execute within their own SAVEPOINT, to ensure data integrity on the updates level. - * - * @returns {Promise} Promise object that resolves with `undefined`. - */ - async applyList(migrationsObj) { + async applyAll(migrationsObj) { const savedMigrations = await this.get({}, { limit: null }); const pendingMigrations = await this.readPending( @@ -397,10 +293,19 @@ class Migration extends BaseEntity { for (const migration of pendingMigrations) { const execute = tx => this.applyPendingMigration(migration, tx); // eslint-disable-next-line no-await-in-loop - await this.begin('migrations:applyList', execute); + await this.begin('migrations:applyAll', execute); } } } + + /** + * Define migrations schema + * + * @returns {Promise} Promise object that resolves with `undefined`. + */ + async defineSchema() { + return this.adapter.executeFile(this.SQLs.defineSchema); + } } module.exports = Migration; diff --git a/framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql b/framework/src/controller/migrations/sql/define_schema.sql similarity index 54% rename from framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql rename to framework/src/controller/migrations/sql/define_schema.sql index f4bbf49eaf4..ab299f6b2da 100644 --- a/framework/src/controller/migrations/sql/updates/20190521000002_migrate_migrations_table.sql +++ b/framework/src/controller/migrations/sql/define_schema.sql @@ -14,16 +14,29 @@ /* - DESCRIPTION: Remove migrations-related entries and migrate existing entries namespace + DESCRIPTION: Creates schema. PARAMETERS: None */ +-- Create migrations table +CREATE TABLE IF NOT EXISTS "migrations"( + "id" VARCHAR(22) NOT NULL PRIMARY KEY, + "name" TEXT NOT NULL +); + +-- Add new column `namespace` to identify migration's groups +ALTER TABLE "migrations" ADD COLUMN IF NOT EXISTS "namespace" TEXT NOT NULL DEFAULT 'undefined'; + +-- Make `namespace` column composite primary key along with `id` +ALTER TABLE "migrations" DROP CONSTRAINT IF EXISTS "migrations_pkey"; +ALTER TABLE "migrations" ADD CONSTRAINT "migrations_pkey" PRIMARY KEY ("id", "namespace"); + -- Remove framework-related migration (20180205000000_underscore_patch.sql) DELETE FROM "migrations" WHERE id = '20180205000000' AND name = 'underscore_patch'; --- Populate `namespace` with correct initial values when exist -UPDATE "migrations" SET namespace = 'network' WHERE id IN ('20161016133824', '20170113181857', '20171207000001', '20171227155620', '20180205000002', '20180327170000', '20181106000006', '20190103000001', '20190111111557'); +-- Populate `namespace` with correct initial values when it exists +UPDATE "migrations" SET namespace = 'network' WHERE namespace = 'undefined' AND id IN ('20161016133824', '20170113181857', '20171207000001', '20171227155620', '20180205000002', '20180327170000', '20181106000006', '20190103000001', '20190111111557'); UPDATE "migrations" SET namespace = 'chain' WHERE namespace = 'undefined'; -- Duplicate 20160723182900_create_schema.sql migration for `network` module if migration exists for `chain` module @@ -32,4 +45,6 @@ INSERT INTO "migrations" ("id", "name", "namespace") SELECT '20160723182900', 'create_schema', 'network' WHERE EXISTS ( SELECT * FROM "migrations" WHERE id = '20160723182900' AND name = 'create_schema' AND namespace = 'chain' +) AND NOT EXISTS ( + SELECT * FROM "migrations" WHERE id = '20160723182900' AND name = 'create_schema' AND namespace = 'network' ); diff --git a/framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql b/framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql deleted file mode 100644 index 95960417eda..00000000000 --- a/framework/src/controller/migrations/sql/updates/20160723182900_create_schema.sql +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright © 2018 Lisk Foundation - * - * See the LICENSE file at the top-level directory of this distribution - * for licensing information. - * - * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, - * no part of this software, including this file, may be copied, modified, - * propagated, or distributed except according to the terms contained in the - * LICENSE file. - * - * Removal or modification of this copyright notice is prohibited. - */ - - -/* - DESCRIPTION: Creates `internal_migrations` table to manage framework migrations - and `migrations` table to manage app/modules migrations. - - PARAMETERS: None -*/ - -/* Tables */ -CREATE TABLE IF NOT EXISTS "internal_migrations"( - "id" VARCHAR(22) NOT NULL PRIMARY KEY, - "name" TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS "migrations"( - "id" VARCHAR(22) NOT NULL PRIMARY KEY, - "name" TEXT NOT NULL -); diff --git a/framework/src/controller/migrations/sql/updates/20180205000000_underscore_patch.sql b/framework/src/controller/migrations/sql/updates/20180205000000_underscore_patch.sql deleted file mode 100644 index e2cacd2d728..00000000000 --- a/framework/src/controller/migrations/sql/updates/20180205000000_underscore_patch.sql +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright © 2018 Lisk Foundation - * - * See the LICENSE file at the top-level directory of this distribution - * for licensing information. - * - * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, - * no part of this software, including this file, may be copied, modified, - * propagated, or distributed except according to the terms contained in the - * LICENSE file. - * - * Removal or modification of this copyright notice is prohibited. - */ - - -/* - DESCRIPTION: Patches any existing database that contains migration names in camel case, - and changes them into low-case underscore. - - PARAMETERS: None -*/ - -UPDATE migrations SET name = lower(regexp_replace(name, E'([A-Z])', E'\_\\1','g')) diff --git a/framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql b/framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql deleted file mode 100644 index 71e7a6a17e3..00000000000 --- a/framework/src/controller/migrations/sql/updates/20190521000001_add_namespace_to_migration.sql +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright © 2018 Lisk Foundation - * - * See the LICENSE file at the top-level directory of this distribution - * for licensing information. - * - * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, - * no part of this software, including this file, may be copied, modified, - * propagated, or distributed except according to the terms contained in the - * LICENSE file. - * - * Removal or modification of this copyright notice is prohibited. - */ - - -/* - DESCRIPTION: Add `namespace` column to migrations table - and replace primary key by a composite key using id and namespace - - PARAMETERS: None -*/ - -ALTER TABLE "migrations" ADD COLUMN IF NOT EXISTS "namespace" TEXT NOT NULL DEFAULT 'undefined'; -ALTER TABLE "migrations" DROP CONSTRAINT IF EXISTS "migrations_pkey"; -ALTER TABLE "migrations" ADD CONSTRAINT IF NOT EXISTS "migrations_pkey" PRIMARY KEY ("id", "namespace"); From 73b4e38d41988f8e200997eaaeb893ac8b018cc9 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Mon, 27 May 2019 16:53:59 +0200 Subject: [PATCH 19/22] Fix unit tests --- framework/test/mocha/common/application.js | 4 +- .../test/mocha/common/storage_sandbox.js | 4 +- .../components/storage/entities/migration.js | 187 ++---------------- 3 files changed, 16 insertions(+), 179 deletions(-) diff --git a/framework/test/mocha/common/application.js b/framework/test/mocha/common/application.js index e1640b8015d..010ffea2bdb 100644 --- a/framework/test/mocha/common/application.js +++ b/framework/test/mocha/common/application.js @@ -116,8 +116,8 @@ async function __init(sandbox, initScope) { }) .then(async status => { if (status) { - await storage.entities.Migration.applyInternal(); - await storage.entities.Migration.applyList(modulesMigrations); + await storage.entities.Migration.defineSchema(); + await storage.entities.Migration.applyAll(modulesMigrations); } }); diff --git a/framework/test/mocha/common/storage_sandbox.js b/framework/test/mocha/common/storage_sandbox.js index f3a2579f24b..f4266af81d6 100644 --- a/framework/test/mocha/common/storage_sandbox.js +++ b/framework/test/mocha/common/storage_sandbox.js @@ -136,8 +136,8 @@ class StorageSandbox extends Storage { async _createSchema() { try { - await this.entities.Migration.applyInternal(); - await this.entities.Migration.applyList(modulesMigrations); + await this.entities.Migration.defineSchema(); + await this.entities.Migration.applyAll(modulesMigrations); } catch (err) { Promise.reject(err); } diff --git a/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js b/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js index 00633deece4..ea2d708bc3d 100644 --- a/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js +++ b/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js @@ -15,8 +15,6 @@ 'use strict'; -const path = require('path'); -const fs = require('fs-extra'); const { entities: { BaseEntity }, errors: { @@ -364,7 +362,6 @@ describe('Migration', () => { }); describe('Schema Updates methods', () => { - let internalMigrationIDs; let savedMigrations; const modulesMigrations = {}; @@ -373,16 +370,6 @@ describe('Migration', () => { modulesMigrations[HttpAPIModule.alias] = HttpAPIModule.migrations; before(async () => { - const internalMigrationFiles = await fs.readdir( - path.join( - __dirname, - '../../../../../../../../src/controller/migrations/sql/updates' - ) - ); - internalMigrationIDs = internalMigrationFiles - .map(f => f.match(/(\d+)_(.+).sql/)[1]) - .sort(); - savedMigrations = Object.keys(modulesMigrations).reduce( (prev, namespace) => { const curr = modulesMigrations[namespace].map(migrationFile => { @@ -405,99 +392,6 @@ describe('Migration', () => { sinonSandbox.restore(); }); - describe('hasInternalMigrationsTable()', () => { - it('should resolve with true if internal_migrations table exists', async () => { - expect(await storage.entities.Migration.hasInternalMigrationsTable()).to - .be.true; - }); - - it('should resolve with false if migrations table does not exists', async () => { - // Create backup table and drop migrations table - await storage.entities.Migration.adapter.execute( - 'CREATE TABLE temp_migrations AS TABLE internal_migrations; DROP TABLE internal_migrations;' - ); - - expect(await storage.entities.Migration.hasInternalMigrationsTable()).to - .be.false; - - // Restore the migrations table and drop the backup - await storage.entities.Migration.adapter.execute( - 'CREATE TABLE internal_migrations AS TABLE temp_migrations; DROP TABLE temp_migrations;' - ); - }); - }); - - describe('hasMigrationsTable()', () => { - it('should resolve with true if migrations table exists', async () => { - expect(await storage.entities.Migration.hasMigrationsTable()).to.be - .true; - }); - - it('should resolve with false if migrations table does not exists', async () => { - // Create backup table and drop migrations table - await storage.entities.Migration.adapter.execute( - 'CREATE TABLE temp_migrations AS TABLE migrations; DROP TABLE migrations;' - ); - - expect(await storage.entities.Migration.hasMigrationsTable()).to.be - .false; - - // Restore the migrations table and drop the backup - await storage.entities.Migration.adapter.execute( - 'CREATE TABLE migrations AS TABLE temp_migrations; DROP TABLE temp_migrations;' - ); - }); - }); - - describe('getLastInternalMigrationId', () => { - it('should use the correct filters for get', async () => { - sinonSandbox.spy(storage.entities.Migration.adapter, 'execute'); - await storage.entities.Migration.getLastInternalMigrationId(); - expect( - storage.entities.Migration.adapter.execute.firstCall.args[0] - ).to.be.eql( - 'SELECT id FROM internal_migrations ORDER BY id DESC LIMIT 1;' - ); - }); - - it('should return id of the last migration file', async () => { - const result = await storage.entities.Migration.getLastInternalMigrationId(); - expect(result).to.be.eql( - internalMigrationIDs[internalMigrationIDs.length - 1] - ); - }); - }); - - describe('readPendingInternalMigrations', () => { - it('should resolve with list of pending files if there exists any', async () => { - const pending = (await storage.entities.Migration.readPendingInternalMigrations( - internalMigrationIDs[0] - )).map(f => f.id); - internalMigrationIDs.splice(0, 1); - expect(pending).to.be.eql(internalMigrationIDs); - }); - - it('should resolve with empty array if there is no pending migration', async () => { - const pending = await storage.entities.Migration.readPendingInternalMigrations( - internalMigrationIDs[internalMigrationIDs.length - 1] - ); - - return expect(pending).to.be.empty; - }); - - it('should resolve with the list in correct format', async () => { - const pending = await storage.entities.Migration.readPendingInternalMigrations( - internalMigrationIDs[0] - ); - - expect(pending).to.be.an('array'); - expect(pending[0]).to.have.all.keys('id', 'name', 'path', 'file'); - expect(pending[0].file).to.be.instanceOf( - storage.entities.Migration.adapter.pgp.QueryFile - ); - }); - }); - describe('readPending', () => { it('should resolve with list of pending files if there exists any', async () => { const pendingMigrationsMock = savedMigrations.slice(0, 2); @@ -545,76 +439,19 @@ describe('Migration', () => { }); }); - describe('applyInternal()', () => { - let updates; - - beforeEach(async () => { - updates = await storage.entities.Migration.readPendingInternalMigrations( - internalMigrationIDs[0] - ); - }); - - it('should call hasMigrations()', async () => { - sinonSandbox.spy( - storage.entities.Migration, - 'hasInternalMigrationsTable' - ); - await storage.entities.Migration.applyInternal(); - expect(storage.entities.Migration.hasInternalMigrationsTable).to.be - .calledOnce; - }); - - it('should call getLastInternalMigrationId()', async () => { - sinonSandbox.spy( - storage.entities.Migration, - 'getLastInternalMigrationId' - ); - await storage.entities.Migration.applyInternal(); - expect(storage.entities.Migration.getLastInternalMigrationId).to.be - .calledOnce; - }); - - it('should call readPendingInternalMigrations()', async () => { - sinonSandbox.spy( - storage.entities.Migration, - 'readPendingInternalMigrations' - ); - await storage.entities.Migration.applyInternal(); - expect(storage.entities.Migration.readPendingInternalMigrations).to.be - .calledOnce; - }); - - it('should apply all pending internal migrations in independent transactions', async () => { - sinonSandbox.spy(storage.entities.Migration, 'begin'); - sinonSandbox - .stub(storage.entities.Migration, 'readPendingInternalMigrations') - .resolves(updates); - sinonSandbox - .stub(storage.entities.Migration, 'applyPendingInternalMigration') - .resolves(null); - - await storage.entities.Migration.applyInternal(); + describe('defineSchema()', () => { + it('should call adapter.executeFile with proper params', async () => { + sinonSandbox.spy(adapter, 'executeFile'); + const migration = new MigrationEntity(adapter); + await migration.defineSchema(); - expect(storage.entities.Migration.begin.callCount).to.be.eql( - updates.length + expect(adapter.executeFile.firstCall.args[0]).to.be.eql( + migration.SQLs.defineSchema ); - expect( - storage.entities.Migration.begin - .getCalls() - .filter(aCall => aCall.args[0] !== 'migrations:applyInternal') - .length - ).to.be.eql(0); - - const applyPendingInternalMigrationCalls = storage.entities.Migration.applyPendingInternalMigration.getCalls(); - - applyPendingInternalMigrationCalls.forEach((aCall, idx) => { - expect(aCall.args[0]).to.be.eql(updates[idx]); - expect(aCall.args[1].constructor.name).to.be.eql('Task'); - }); }); }); - describe('applyList()', () => { + describe('applyAll()', () => { let pendingMigrations; beforeEach(async () => { @@ -627,13 +464,13 @@ describe('Migration', () => { it('should call this.get()', async () => { sinonSandbox.spy(storage.entities.Migration, 'get'); - await storage.entities.Migration.applyList(modulesMigrations); + await storage.entities.Migration.applyAll(modulesMigrations); expect(storage.entities.Migration.get).to.be.calledOnce; }); it('should call readPending()', async () => { sinonSandbox.spy(storage.entities.Migration, 'readPending'); - await storage.entities.Migration.applyList(modulesMigrations); + await storage.entities.Migration.applyAll(modulesMigrations); expect(storage.entities.Migration.readPending).to.be.calledOnce; }); @@ -646,7 +483,7 @@ describe('Migration', () => { .stub(storage.entities.Migration, 'applyPendingMigration') .resolves(null); - await storage.entities.Migration.applyList(modulesMigrations); + await storage.entities.Migration.applyAll(modulesMigrations); expect(storage.entities.Migration.begin.callCount).to.be.eql( pendingMigrations.length @@ -654,7 +491,7 @@ describe('Migration', () => { expect( storage.entities.Migration.begin .getCalls() - .filter(aCall => aCall.args[0] !== 'migrations:applyList').length + .filter(aCall => aCall.args[0] !== 'migrations:applyAll').length ).to.be.eql(0); const applyPendingMigrationCalls = storage.entities.Migration.applyPendingMigration.getCalls(); From ff109d51bc50a2bf734f157f244b01220dbe2900 Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Mon, 27 May 2019 17:16:39 +0200 Subject: [PATCH 20/22] Improve method description text --- framework/src/controller/migrations/migration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration.js index 3bcc32df7d5..66219291c51 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration.js @@ -275,7 +275,7 @@ class Migration extends BaseEntity { } /** - * Applies a cumulative update: all pending migrations + runtime. + * Applies a cumulative update: all migrations passed as argument except the ones present in the migrations table. * Each update+insert execute within their own SAVEPOINT, to ensure data integrity on the updates level. * * @returns {Promise} Promise object that resolves with `undefined`. From 4d7c8931fcdec357ee155148456d25f092711b1e Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Tue, 28 May 2019 14:16:09 +0200 Subject: [PATCH 21/22] Rename migration entity --- framework/src/controller/migrations/index.js | 2 +- .../migrations/{migration.js => migration_entity.js} | 4 ++-- .../modules/chain/components/storage/entities/migration.js | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) rename framework/src/controller/migrations/{migration.js => migration_entity.js} (99%) diff --git a/framework/src/controller/migrations/index.js b/framework/src/controller/migrations/index.js index bd7012fecb5..76e97ccfbc8 100644 --- a/framework/src/controller/migrations/index.js +++ b/framework/src/controller/migrations/index.js @@ -14,7 +14,7 @@ 'use strict'; -const MigrationEntity = require('./migration'); +const MigrationEntity = require('./migration_entity'); module.exports = { MigrationEntity, diff --git a/framework/src/controller/migrations/migration.js b/framework/src/controller/migrations/migration_entity.js similarity index 99% rename from framework/src/controller/migrations/migration.js rename to framework/src/controller/migrations/migration_entity.js index 66219291c51..d16fc5d3ee1 100644 --- a/framework/src/controller/migrations/migration.js +++ b/framework/src/controller/migrations/migration_entity.js @@ -62,7 +62,7 @@ const sqlFiles = { * @property {string} [namespace_like] */ -class Migration extends BaseEntity { +class MigrationEntity extends BaseEntity { /** * Constructor * @param {BaseAdapter} adapter - Adapter to retrieve the data from @@ -308,4 +308,4 @@ class Migration extends BaseEntity { } } -module.exports = Migration; +module.exports = MigrationEntity; diff --git a/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js b/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js index ea2d708bc3d..efde69133c9 100644 --- a/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js +++ b/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js @@ -114,7 +114,9 @@ describe('Migration', () => { it('should be a constructable function', async () => { expect(MigrationEntity.prototype.constructor).not.to.be.null; - expect(MigrationEntity.prototype.constructor.name).to.be.eql('Migration'); + expect(MigrationEntity.prototype.constructor.name).to.be.eql( + 'MigrationEntity' + ); }); it('should extend BaseEntity', async () => { From f9cd53032ff5f43da08471e45d0e37efadaf8ceb Mon Sep 17 00:00:00 2001 From: Lucas Silvestre Date: Tue, 28 May 2019 14:24:06 +0200 Subject: [PATCH 22/22] Remove undefined as default value for namespace --- .../controller/migrations/sql/define_schema.sql | 14 +++++++------- .../chain/components/storage/entities/migration.js | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/framework/src/controller/migrations/sql/define_schema.sql b/framework/src/controller/migrations/sql/define_schema.sql index ab299f6b2da..352590e69d8 100644 --- a/framework/src/controller/migrations/sql/define_schema.sql +++ b/framework/src/controller/migrations/sql/define_schema.sql @@ -26,18 +26,18 @@ CREATE TABLE IF NOT EXISTS "migrations"( ); -- Add new column `namespace` to identify migration's groups -ALTER TABLE "migrations" ADD COLUMN IF NOT EXISTS "namespace" TEXT NOT NULL DEFAULT 'undefined'; - --- Make `namespace` column composite primary key along with `id` -ALTER TABLE "migrations" DROP CONSTRAINT IF EXISTS "migrations_pkey"; -ALTER TABLE "migrations" ADD CONSTRAINT "migrations_pkey" PRIMARY KEY ("id", "namespace"); +ALTER TABLE "migrations" ADD COLUMN IF NOT EXISTS "namespace" TEXT; -- Remove framework-related migration (20180205000000_underscore_patch.sql) DELETE FROM "migrations" WHERE id = '20180205000000' AND name = 'underscore_patch'; -- Populate `namespace` with correct initial values when it exists -UPDATE "migrations" SET namespace = 'network' WHERE namespace = 'undefined' AND id IN ('20161016133824', '20170113181857', '20171207000001', '20171227155620', '20180205000002', '20180327170000', '20181106000006', '20190103000001', '20190111111557'); -UPDATE "migrations" SET namespace = 'chain' WHERE namespace = 'undefined'; +UPDATE "migrations" SET namespace = 'network' WHERE namespace IS NULL AND id IN ('20161016133824', '20170113181857', '20171207000001', '20171227155620', '20180205000002', '20180327170000', '20181106000006', '20190103000001', '20190111111557'); +UPDATE "migrations" SET namespace = 'chain' WHERE namespace IS NULL; + +-- Make `namespace` column composite primary key along with `id` +ALTER TABLE "migrations" DROP CONSTRAINT IF EXISTS "migrations_pkey"; +ALTER TABLE "migrations" ADD CONSTRAINT "migrations_pkey" PRIMARY KEY ("id", "namespace"); -- Duplicate 20160723182900_create_schema.sql migration for `network` module if migration exists for `chain` module -- That is required because 20160723182900_create_schema.sql was splited into two different files (one for each module) diff --git a/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js b/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js index efde69133c9..2374eb0118c 100644 --- a/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js +++ b/framework/test/mocha/unit/modules/chain/components/storage/entities/migration.js @@ -101,6 +101,7 @@ describe('Migration', () => { validMigration = { id: '30200723182900', name: 'create_schema', + namespace: 'module_name', }; adapter = storage.adapter;