From 2183cd19f398ef59d7ec0d4838af7da7605d4aa1 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Sat, 12 Oct 2019 18:34:16 +0300 Subject: [PATCH 1/2] changed the "migrate_only_data" configuration parameter to boolean --- config/config.json | 5 ++--- src/ConstraintsProcessor.ts | 23 ++++++++++++++++------- src/Conversion.ts | 13 ++++++------- src/DataLoader.ts | 4 ++-- src/MigrationStateManager.ts | 7 ++++--- src/TableProcessor.ts | 2 +- 6 files changed, 31 insertions(+), 23 deletions(-) diff --git a/config/config.json b/config/config.json index 2aa221ef..63981d08 100644 --- a/config/config.json +++ b/config/config.json @@ -83,10 +83,9 @@ "migrate_only_data_description" : [ "In order to skip schema migration, and just migrate data into a preset tables", - "fill following array with preset table names.", - "If all of your tables are preset, then use '*'" + " - set this parameter true." ], - "migrate_only_data" : [], + "migrate_only_data" : false, "delimiter_description" : [ "Specifies the character, that separates columns within each record.", diff --git a/src/ConstraintsProcessor.ts b/src/ConstraintsProcessor.ts index d99fd629..1dd13de6 100644 --- a/src/ConstraintsProcessor.ts +++ b/src/ConstraintsProcessor.ts @@ -37,10 +37,11 @@ import Conversion from './Conversion'; */ export default async function(conversion: Conversion): Promise { const isTableConstraintsLoaded: boolean = await migrationStateManager.get(conversion, 'per_table_constraints_loaded'); + const migrateOnlyData: boolean = conversion.shouldMigrateOnlyData(); const promises: Promise[] = conversion._tablesToMigrate.map(async (tableName: string) => { if (!isTableConstraintsLoaded) { - if (conversion.shouldMigrateOnlyDataFor(tableName)) { + if (migrateOnlyData) { return sequencesProcessor.setSequenceValue(conversion, tableName); } @@ -54,13 +55,21 @@ export default async function(conversion: Conversion): Promise { }); await Promise.all(promises); - await migrationStateManager.set(conversion, 'per_table_constraints_loaded'); - await processForeignKey(conversion); - await migrationStateManager.set(conversion, 'foreign_keys_loaded'); + + if (migrateOnlyData) { + await migrationStateManager.set(conversion, 'per_table_constraints_loaded', 'foreign_keys_loaded', 'views_loaded'); + } else { + await migrationStateManager.set(conversion, 'per_table_constraints_loaded'); + await processForeignKey(conversion); + await migrationStateManager.set(conversion, 'foreign_keys_loaded'); + await processViews(conversion); + await migrationStateManager.set(conversion, 'views_loaded'); + } + + await runVacuumFullAndAnalyze(conversion); // Reclaim storage occupied by dead tuples. + + // !!!Note, dropping of data-pool and state-logs tables MUST be the last step of migration process. await dataPoolManager.dropDataPoolTable(conversion); - await processViews(conversion); - await migrationStateManager.set(conversion, 'views_loaded'); - await runVacuumFullAndAnalyze(conversion); await migrationStateManager.dropStateLogsTable(conversion); generateReport(conversion, 'NMIG migration is accomplished.'); } diff --git a/src/Conversion.ts b/src/Conversion.ts index e84cc931..a4e4f76b 100644 --- a/src/Conversion.ts +++ b/src/Conversion.ts @@ -71,10 +71,9 @@ export default class Conversion { public readonly _delimiter: string; /** - * Defines preset tables. - * The only thing to do with these tables is a data migration, since the schema is preset. + * Indicates if the schema in the target database is preset. */ - public readonly _migrateOnlyData: string[]; + public readonly _migrateOnlyData: boolean; /** * A path to the "logs_directory". @@ -231,17 +230,17 @@ export default class Conversion { this._maxDbConnectionPoolSize = this._maxDbConnectionPoolSize > 0 ? this._maxDbConnectionPoolSize : 10; this._loaderMaxOldSpaceSize = this._config.loader_max_old_space_size; this._loaderMaxOldSpaceSize = Conversion._isIntNumeric(this._loaderMaxOldSpaceSize) ? this._loaderMaxOldSpaceSize : 'DEFAULT'; - this._migrateOnlyData = this._config.migrate_only_data === undefined ? [] : this._config.migrate_only_data; + this._migrateOnlyData = this._config.migrate_only_data === undefined ? false : this._config.migrate_only_data; this._delimiter = this._config.delimiter !== undefined && this._config.delimiter.length === 1 ? this._config.delimiter : ','; } /** - * Checks if there are actions to take on given table other than data migration. + * Checks if there are actions to take other than data migration. */ - public shouldMigrateOnlyDataFor(tableName: string): boolean { - return this._migrateOnlyData.indexOf(tableName) !== -1 || this._migrateOnlyData.indexOf('*') !== -1; + public shouldMigrateOnlyData(): boolean { + return this._migrateOnlyData; } /** diff --git a/src/DataLoader.ts b/src/DataLoader.ts index 85cb01d4..ad49a04a 100644 --- a/src/DataLoader.ts +++ b/src/DataLoader.ts @@ -69,7 +69,6 @@ async function deleteChunk( originalSessionReplicationRole: string | null = null ): Promise { const sql: string = `DELETE FROM "${ conversion._schema }"."data_pool_${ conversion._schema }${ conversion._mySqlDbName }" WHERE id = ${ dataPoolId };`; - const dbAccess: DBAccess = new DBAccess(conversion); try { await client.query(sql); @@ -80,6 +79,7 @@ async function deleteChunk( } catch (error) { await generateError(conversion, `\t--[DataLoader::deleteChunk] ${ error }`, sql); } finally { + const dbAccess: DBAccess = new DBAccess(conversion); await dbAccess.releaseDbClient(client); } } @@ -122,7 +122,7 @@ async function populateTableWorker( const client: PoolClient = await dbAccess.getPgClient(); let originalSessionReplicationRole: string | null = null; - if (conv.shouldMigrateOnlyDataFor(tableName)) { + if (conv.shouldMigrateOnlyData()) { originalSessionReplicationRole = await disableTriggers(conv, client); } diff --git a/src/MigrationStateManager.ts b/src/MigrationStateManager.ts index aa86ca7a..8029d0b1 100644 --- a/src/MigrationStateManager.ts +++ b/src/MigrationStateManager.ts @@ -37,9 +37,10 @@ export async function get(conversion: Conversion, param: string): Promise { +export async function set(conversion: Conversion, ...states: string[]): Promise { + const statesSql: string = states.map((state: string) => `${ state } = TRUE`).join(','); + const sql: string = `UPDATE "${ conversion._schema }"."state_logs_${ conversion._schema }${ conversion._mySqlDbName }" SET ${ statesSql };`; const dbAccess: DBAccess = new DBAccess(conversion); - const sql: string = `UPDATE "${ conversion._schema }"."state_logs_${ conversion._schema }${ conversion._mySqlDbName }" SET ${ param } = TRUE;`; await dbAccess.query('MigrationStateManager::set', sql, DBVendors.PG, true, false); } @@ -57,7 +58,7 @@ export async function createStateLogsTable(conversion: Conversion): Promise Date: Sat, 12 Oct 2019 20:03:05 +0300 Subject: [PATCH 2/2] updated nmig version --- README.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 21d20af7..90f6f8de 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ from MySQL to PostgreSQL as easy and smooth as possible.

Note: "logs_directory" will be created during script execution.

VERSION

-

Current version is 5.0.0
+

Current version is 5.0.1
(major version . improvements . bug fixes)

KNOWN ISSUES

diff --git a/package.json b/package.json index 542304df..b7e254b2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "nmig", - "version": "5.0.0", + "version": "5.0.1", "description": "The database migration app", "author": "Anatoly Khaytovich", "license": "GPL-3.0",