Merge branch 'feature/fix-data-transfer-into-preset-schema' into develop

README.md

@@ -87,7 +87,7 @@ from MySQL to PostgreSQL as easy and smooth as possible.</p>
 <b>Note:</b> "logs_directory" will be created during script execution.</p>
 
 <h3>VERSION</h3>
-<p>Current version is 5.0.0<br />
+<p>Current version is 5.0.1<br />
 (major version . improvements . bug fixes)</p>
 
 <h3>KNOWN ISSUES</h3>

config/config.json

@@ -83,10 +83,9 @@
 
     "migrate_only_data_description" : [
         "In order to skip schema migration, and just migrate data into a preset tables",
-        "fill following array with preset table names.",
-        "If all of your tables are preset, then use '*'"
+        " - set this parameter true."
     ],
-    "migrate_only_data" : [],
+    "migrate_only_data" : false,
 
     "delimiter_description" : [
         "Specifies the character, that separates columns within each record.",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "nmig",
-    "version": "5.0.0",
+    "version": "5.0.1",
     "description": "The database migration app",
     "author": "Anatoly Khaytovich<[email protected]>",
     "license": "GPL-3.0",

src/ConstraintsProcessor.ts

@@ -37,10 +37,11 @@ import Conversion from './Conversion';
  */
 export default async function(conversion: Conversion): Promise<void> {
     const isTableConstraintsLoaded: boolean = await migrationStateManager.get(conversion, 'per_table_constraints_loaded');
+    const migrateOnlyData: boolean = conversion.shouldMigrateOnlyData();
 
     const promises: Promise<void>[] = conversion._tablesToMigrate.map(async (tableName: string) => {
         if (!isTableConstraintsLoaded) {
-            if (conversion.shouldMigrateOnlyDataFor(tableName)) {
+            if (migrateOnlyData) {
                 return sequencesProcessor.setSequenceValue(conversion, tableName);
             }
 
@@ -54,13 +55,21 @@ export default async function(conversion: Conversion): Promise<void> {
     });
 
     await Promise.all(promises);
-    await migrationStateManager.set(conversion, 'per_table_constraints_loaded');
-    await processForeignKey(conversion);
-    await migrationStateManager.set(conversion, 'foreign_keys_loaded');
+
+    if (migrateOnlyData) {
+        await migrationStateManager.set(conversion, 'per_table_constraints_loaded', 'foreign_keys_loaded', 'views_loaded');
+    } else {
+        await migrationStateManager.set(conversion, 'per_table_constraints_loaded');
+        await processForeignKey(conversion);
+        await migrationStateManager.set(conversion, 'foreign_keys_loaded');
+        await processViews(conversion);
+        await migrationStateManager.set(conversion, 'views_loaded');
+    }
+
+    await runVacuumFullAndAnalyze(conversion); // Reclaim storage occupied by dead tuples.
+
+    // !!!Note, dropping of data-pool and state-logs tables MUST be the last step of migration process.
     await dataPoolManager.dropDataPoolTable(conversion);
-    await processViews(conversion);
-    await migrationStateManager.set(conversion, 'views_loaded');
-    await runVacuumFullAndAnalyze(conversion);
     await migrationStateManager.dropStateLogsTable(conversion);
     generateReport(conversion, 'NMIG migration is accomplished.');
 }

src/Conversion.ts

@@ -71,10 +71,9 @@ export default class Conversion {
     public readonly _delimiter: string;
 
     /**
-     * Defines preset tables.
-     * The only thing to do with these tables is a data migration, since the schema is preset.
+     * Indicates if the schema in the target database is preset.
      */
-    public readonly _migrateOnlyData: string[];
+    public readonly _migrateOnlyData: boolean;
 
     /**
      * A path to the "logs_directory".
@@ -231,17 +230,17 @@ export default class Conversion {
         this._maxDbConnectionPoolSize = this._maxDbConnectionPoolSize > 0 ? this._maxDbConnectionPoolSize : 10;
         this._loaderMaxOldSpaceSize = this._config.loader_max_old_space_size;
         this._loaderMaxOldSpaceSize = Conversion._isIntNumeric(this._loaderMaxOldSpaceSize) ? this._loaderMaxOldSpaceSize : 'DEFAULT';
-        this._migrateOnlyData = this._config.migrate_only_data === undefined ? [] : this._config.migrate_only_data;
+        this._migrateOnlyData = this._config.migrate_only_data === undefined ? false : this._config.migrate_only_data;
         this._delimiter = this._config.delimiter !== undefined && this._config.delimiter.length === 1
             ? this._config.delimiter
             : ',';
     }
 
     /**
-     * Checks if there are actions to take on given table other than data migration.
+     * Checks if there are actions to take other than data migration.
      */
-    public shouldMigrateOnlyDataFor(tableName: string): boolean {
-        return this._migrateOnlyData.indexOf(tableName) !== -1 || this._migrateOnlyData.indexOf('*') !== -1;
+    public shouldMigrateOnlyData(): boolean {
+        return this._migrateOnlyData;
     }
 
     /**

src/DataLoader.ts

@@ -69,7 +69,6 @@ async function deleteChunk(
     originalSessionReplicationRole: string | null = null
 ): Promise<void> {
     const sql: string = `DELETE FROM "${ conversion._schema }"."data_pool_${ conversion._schema }${ conversion._mySqlDbName }" WHERE id = ${ dataPoolId };`;
-    const dbAccess: DBAccess = new DBAccess(conversion);
 
     try {
         await client.query(sql);
@@ -80,6 +79,7 @@ async function deleteChunk(
     } catch (error) {
         await generateError(conversion, `\t--[DataLoader::deleteChunk] ${ error }`, sql);
     } finally {
+        const dbAccess: DBAccess = new DBAccess(conversion);
         await dbAccess.releaseDbClient(client);
     }
 }
@@ -122,7 +122,7 @@ async function populateTableWorker(
     const client: PoolClient = await dbAccess.getPgClient();
     let originalSessionReplicationRole: string | null = null;
 
-    if (conv.shouldMigrateOnlyDataFor(tableName)) {
+    if (conv.shouldMigrateOnlyData()) {
         originalSessionReplicationRole = await disableTriggers(conv, client);
     }
 

src/MigrationStateManager.ts

@@ -37,9 +37,10 @@ export async function get(conversion: Conversion, param: string): Promise<boolea
 /**
  * Updates the state-log.
  */
-export async function set(conversion: Conversion, param: string): Promise<void> {
+export async function set(conversion: Conversion, ...states: string[]): Promise<void> {
+    const statesSql: string = states.map((state: string) => `${ state } = TRUE`).join(',');
+    const sql: string = `UPDATE "${ conversion._schema }"."state_logs_${ conversion._schema }${ conversion._mySqlDbName }" SET ${ statesSql };`;
     const dbAccess: DBAccess = new DBAccess(conversion);
-    const sql: string = `UPDATE "${ conversion._schema }"."state_logs_${ conversion._schema }${ conversion._mySqlDbName }" SET ${ param } = TRUE;`;
     await dbAccess.query('MigrationStateManager::set', sql, DBVendors.PG, true, false);
 }
 
@@ -57,7 +58,7 @@ export async function createStateLogsTable(conversion: Conversion): Promise<Conv
 
     if (+result.data.rows[0].cnt === 0) {
         sql = `INSERT INTO "${ conversion._schema }"."state_logs_${ conversion._schema }${ conversion._mySqlDbName }" VALUES (FALSE, FALSE, FALSE, FALSE);`;
-        await await dbAccess.query('MigrationStateManager::createStateLogsTable', sql, DBVendors.PG, true, false, result.client);
+        await dbAccess.query('MigrationStateManager::createStateLogsTable', sql, DBVendors.PG, true, false, result.client);
         return conversion;
     }
 

src/TableProcessor.ts

@@ -87,7 +87,7 @@ export async function createTable(conversion: Conversion, tableName: string): Pr
 
     conversion._dicTables[tableName].arrTableColumns = columns.data;
 
-    if (conversion.shouldMigrateOnlyDataFor(tableName)) {
+    if (conversion.shouldMigrateOnlyData()) {
         return;
     }