Add capability to register database migration from the application/module layer - Closes #3197

framework/src/controller/application.js

@@ -40,6 +40,7 @@ const NetworkModule = require('../modules/network');
 const __private = {
 	modules: new WeakMap(),
 	transactions: new WeakMap(),
+	migrations: new WeakMap(),
 };
 
 const registerProcessHooks = app => {
@@ -152,6 +153,7 @@ class Application {
 
 		__private.modules.set(this, {});
 		__private.transactions.set(this, {});
+		__private.migrations.set(this, {});
 
 		const { TRANSACTION_TYPES } = constants;
 
@@ -205,6 +207,9 @@ class Application {
 			options
 		);
 		__private.modules.set(this, modules);
+
+		// Register migrations defined by the module
+		this.registerMigrations(moduleKlass.alias, moduleKlass.migrations);
 	}
 
 	/**
@@ -255,6 +260,25 @@ class Application {
 		__private.transactions.set(this, transactions);
 	}
 
+	/**
+	 * Register migrations with the application
+	 *
+	 * @param {Object} namespace - Migration namespace
+	 * @param {Array} migrations - Migrations list. Format ['/path/to/migration/yyyyMMddHHmmss_name_of_migration.sql']
+	 */
+	registerMigrations(namespace, migrations) {
+		assert(namespace, 'Namespace is required');
+		assert(migrations instanceof Array, 'Migrations list should be an array');
+		assert(
+			!Object.keys(this.getMigrations()).includes(namespace),
+			`Migrations for "${namespace}" was already registered.`
+		);
+
+		const currentMigrations = this.getMigrations();
+		currentMigrations[namespace] = Object.freeze(migrations);
+		__private.migrations.set(this, currentMigrations);
+	}
+
 	/**
 	 * Get list of all transactions registered with the application
 	 *
@@ -293,6 +317,15 @@ class Application {
 		return __private.modules.get(this);
 	}
 
+	/**
+	 * Get all registered migrations
+	 *
+	 * @return {Array.<Object>}
+	 */
+	getMigrations() {
+		return __private.migrations.get(this);
+	}
+
 	/**
 	 * Run the application
 	 *
@@ -322,7 +355,11 @@ class Application {
 			},
 			this.logger
 		);
-		return this.controller.load(this.getModules(), this.config.modules);
+		return this.controller.load(
+			this.getModules(),
+			this.config.modules,
+			this.getMigrations()
+		);
 	}
 
 	/**

framework/src/controller/controller.js

@@ -24,6 +24,8 @@ const Bus = require('./bus');
 const { DuplicateAppInstanceError } = require('../errors');
 const { validateModuleSpec } = require('./validator');
 const ApplicationState = require('./application_state');
+const { createStorageComponent } = require('../components/storage');
+const { MigrationEntity } = require('./migrations');
 
 const isPidRunning = async pid =>
 	psList().then(list => list.some(x => x.pid === pid));
@@ -69,6 +71,10 @@ class Controller {
 		this.childrenList = [];
 		this.channel = null; // Channel for controller
 		this.bus = null;
+
+		const storageConfig = config.components.storage;
+		this.storage = createStorageComponent(storageConfig, logger);
+		this.storage.registerEntity('Migration', MigrationEntity);
 	}
 
 	/**
@@ -78,12 +84,13 @@ class Controller {
 	 * @param modules
 	 * @async
 	 */
-	async load(modules, moduleOptions) {
+	async load(modules, moduleOptions, migrations = {}) {
 		this.logger.info('Loading controller');
 		await this._setupDirectories();
 		await this._validatePidFile();
 		await this._initState();
 		await this._setupBus();
+		await this._loadMigrations(migrations);
 		await this._loadModules(modules, moduleOptions);
 
 		this.logger.info('Bus listening to events', this.bus.getEvents());
@@ -186,6 +193,12 @@ class Controller {
 		}
 	}
 
+	async _loadMigrations(migrationsObj) {
+		await this.storage.bootstrap();
+		await this.storage.entities.Migration.defineSchema();
+		return this.storage.entities.Migration.applyAll(migrationsObj);
+	}
+
 	async _loadModules(modules, moduleOptions) {
 		// To perform operations in sequence and not using bluebird
 		// eslint-disable-next-line no-restricted-syntax

framework/src/controller/migrations/index.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright © 2018 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+
+'use strict';
+
+const MigrationEntity = require('./migration_entity');
+
+module.exports = {
+	MigrationEntity,
+};

framework/src/modules/chain/components/storage/entities/migration.js → framework/src/controller/migrations/migration_entity.js

@@ -23,21 +23,23 @@ const {
 	utils: {
 		filterTypes: { TEXT },
 	},
-} = require('../../../../../components/storage');
+} = require('../../components/storage');
 
 const defaultCreateValues = {};
 
 const sqlFiles = {
-	select: 'migrations/get.sql',
-	isPersisted: 'migrations/is_persisted.sql',
-	create: 'migrations/create.sql',
+	select: 'get.sql',
+	isPersisted: 'is_persisted.sql',
+	create: 'create.sql',
+	defineSchema: 'define_schema.sql',
 };
 
 /**
  * Migration
  * @typedef {Object} Migration
  * @property {string} id
  * @property {string} name
+ * @property {string} namespace
  */
 
 /**
@@ -53,9 +55,14 @@ const sqlFiles = {
  * @property {string} [name_ne]
  * @property {string} [name_in]
  * @property {string} [name_like]
+ * @property {string} [namespace]
+ * @property {string} [namespace_eql]
+ * @property {string} [namespace_ne]
+ * @property {string} [namespace_in]
+ * @property {string} [namespace_like]
  */
 
-class Migration extends BaseEntity {
+class MigrationEntity extends BaseEntity {
 	/**
 	 * Constructor
 	 * @param {BaseAdapter} adapter - Adapter to retrieve the data from
@@ -66,12 +73,12 @@ class Migration extends BaseEntity {
 
 		this.addField('id', 'string', { filter: TEXT });
 		this.addField('name', 'string', { filter: TEXT });
+		this.addField('namespace', 'string', { filter: TEXT });
 
 		const defaultSort = { sort: 'id:asc' };
 		this.extendDefaultOptions(defaultSort);
 
-		this.sqlDirectory = path.join(path.dirname(__filename), '../sql');
-
+		this.sqlDirectory = path.join(path.dirname(__filename), './sql');
 		this.SQLs = this.loadSQLFiles('migration', sqlFiles, this.sqlDirectory);
 	}
 
@@ -214,85 +221,72 @@ class Migration extends BaseEntity {
 	}
 
 	/**
-	 * Verifies presence of the 'migrations' OID named relation.
-	 *
-	 * @returns {Promise<boolean>} Promise object that resolves with a boolean.
-	 */
-	async hasMigrations() {
-		const hasMigrations = await this.adapter.execute(
-			"SELECT table_name hasMigrations FROM information_schema.tables WHERE table_name = 'migrations';"
-		);
-		return !!hasMigrations.length;
-	}
-
-	/**
-	 * Gets id of the last migration record, or 0, if none exist.
-	 *
-	 * @returns {Promise<number>}
-	 * Promise object that resolves with either 0 or id of the last migration record.
-	 */
-	async getLastId() {
-		const result = await this.get({}, { sort: 'id:DESC', limit: 1 });
-		return result.length ? parseInt(result[0].id) : null;
-	}
-
-	/**
-	 * Reads 'sql/migrations/updates' folder and returns an array of objects for further processing.
+	 * Creates an array of objects with `{id, name, namespace, path}`, remove the ones already executed, sorts by ID ascending and add the file property
 	 *
-	 * @param {number} lastMigrationId
+	 * @param {Objec} migrationsObj - Object where the key is the migrations namespace and the value an array of migration's path
+	 * @param {Array} savedMigrations - Array of objects with all migrations already executed before
 	 * @returns {Promise<Array<Object>>}
-	 * Promise object that resolves with an array of objects `{id, name, path, file}`.
+	 * Promise object that resolves with an array of objects `{id, name, namespace, path, file}`.
 	 */
-	readPending(lastMigrationId) {
-		const updatesPath = path.join(__dirname, '../sql/migrations/updates');
-		return fs.readdir(updatesPath).then(files =>
-			files
+	async readPending(migrationsObj, savedMigrations) {
+		return Object.keys(migrationsObj).reduce((prev, namespace) => {
+			const curr = migrationsObj[namespace]
 				.map(migrationFile => {
 					const migration = migrationFile.match(/(\d+)_(.+).sql/);
 					return (
 						migration && {
 							id: migration[1],
 							name: migration[2],
-							path: path.join('../sql/migrations/updates', migrationFile),
+							path: migrationFile,
+							namespace,
 						}
 					);
 				})
-				.sort((a, b) => a.id - b.id) // Sort by migration ID, ascending
 				.filter(
 					migration =>
 						migration &&
-						fs
-							.statSync(path.join(this.sqlDirectory, migration.path))
-							.isFile() &&
-						(!lastMigrationId || +migration.id > lastMigrationId)
+						fs.statSync(migration.path).isFile() &&
+						!savedMigrations.find(
+							saved =>
+								saved.id === migration.id &&
+								saved.namespace === migration.namespace
+						)
 				)
-				.map(f => {
-					f.file = this.adapter.loadSQLFile(f.path, this.sqlDirectory);
-					return f;
-				})
-		);
+				.sort((a, b) => a.id - b.id) // Sort by migration ID, ascending
+				.map(migration => {
+					migration.file = this.adapter.loadSQLFile(migration.path, '');
+					return migration;
+				});
+			return prev.concat(curr);
+		}, []);
 	}
 
 	async applyPendingMigration(pendingMigration, tx) {
-		// eslint-disable-next-line no-restricted-syntax
 		await this.adapter.executeFile(pendingMigration.file, {}, {}, tx);
 		await this.create(
-			{ id: pendingMigration.id, name: pendingMigration.name },
+			{
+				id: pendingMigration.id,
+				name: pendingMigration.name,
+				namespace: pendingMigration.namespace,
+			},
 			{},
 			tx
 		);
 	}
 
 	/**
-	 * Applies a cumulative update: all pending migrations + runtime.
+	 * Applies a cumulative update: all migrations passed as argument except the ones present in the migrations table.
 	 * Each update+insert execute within their own SAVEPOINT, to ensure data integrity on the updates level.
 	 *
 	 * @returns {Promise} Promise object that resolves with `undefined`.
 	 */
-	async applyAll() {
-		const hasMigrations = await this.hasMigrations();
-		const lastId = hasMigrations ? await this.getLastId() : 0;
-		const pendingMigrations = await this.readPending(lastId);
+	async applyAll(migrationsObj) {
+		const savedMigrations = await this.get({}, { limit: null });
+
+		const pendingMigrations = await this.readPending(
+			migrationsObj,
+			savedMigrations
+		);
 
 		if (pendingMigrations.length > 0) {
 			// eslint-disable-next-line no-restricted-syntax
@@ -303,6 +297,15 @@ class Migration extends BaseEntity {
 			}
 		}
 	}
+
+	/**
+	 * Define migrations schema
+	 *
+	 * @returns {Promise} Promise object that resolves with `undefined`.
+	 */
+	async defineSchema() {
+		return this.adapter.executeFile(this.SQLs.defineSchema);
+	}
 }
 
-module.exports = Migration;
+module.exports = MigrationEntity;

framework/src/controller/migrations/sql/define_schema.sql

@@ -0,0 +1,50 @@
+/*
+ * Copyright © 2018 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+
+
+/*
+  DESCRIPTION: Creates schema.
+
+  PARAMETERS: None
+*/
+
+-- Create migrations table
+CREATE TABLE IF NOT EXISTS "migrations"(
+  "id" VARCHAR(22) NOT NULL PRIMARY KEY,
+  "name" TEXT NOT NULL
+);
+
+-- Add new column `namespace` to identify migration's groups
+ALTER TABLE "migrations" ADD COLUMN IF NOT EXISTS "namespace" TEXT;
+
+-- Remove framework-related migration (20180205000000_underscore_patch.sql)
+DELETE FROM "migrations" WHERE id = '20180205000000' AND name = 'underscore_patch';
+
+-- Populate `namespace` with correct initial values when it exists
+UPDATE "migrations" SET namespace = 'network' WHERE namespace IS NULL AND id IN ('20161016133824', '20170113181857', '20171207000001', '20171227155620', '20180205000002', '20180327170000', '20181106000006', '20190103000001', '20190111111557');
+UPDATE "migrations" SET namespace = 'chain' WHERE namespace IS NULL;
+
+-- Make `namespace` column composite primary key along with `id`
+ALTER TABLE "migrations" DROP CONSTRAINT IF EXISTS "migrations_pkey";
+ALTER TABLE "migrations" ADD CONSTRAINT "migrations_pkey" PRIMARY KEY ("id", "namespace");
+
+-- Duplicate 20160723182900_create_schema.sql migration for `network` module if migration exists for `chain` module
+-- That is required because 20160723182900_create_schema.sql was splited into two different files (one for each module)
+INSERT INTO "migrations" ("id", "name", "namespace")
+SELECT '20160723182900', 'create_schema', 'network'
+WHERE EXISTS (
+  SELECT * FROM "migrations" WHERE id = '20160723182900' AND name = 'create_schema' AND namespace = 'chain'
+) AND NOT EXISTS (
+  SELECT * FROM "migrations" WHERE id = '20160723182900' AND name = 'create_schema' AND namespace = 'network'
+);