-
Notifications
You must be signed in to change notification settings - Fork 84
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature/general improvements 1 (#91)
* added number_of_simultaneously_running_loader_processes config parameter * refactoring * improved getNumberOfSimultaneouslyRunningLoaderProcesses function * create identity for auto-incremented columns instead of plain sequences * fixed DefaultProcessor.ts * updated dependencies * fixed TypeScript minor compilation errors * improved data loading performance * small refactoring * updated dependencies Co-authored-by: Anatoly Khaytovich <[email protected]>
- Loading branch information
1 parent
0463549
commit c7bb43c
Showing
21 changed files
with
1,822 additions
and
419 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| { | ||
| "name": "nmig", | ||
| "version": "5.5.0", | ||
| "version": "5.6.0", | ||
| "description": "The database migration app", | ||
| "author": "Anatoly Khaytovich<[email protected]>", | ||
| "license": "GPL-3.0", | ||
|
|
@@ -12,21 +12,21 @@ | |
| "node": ">=10.0.0" | ||
| }, | ||
| "dependencies": { | ||
| "json2csv": "^5.0.3", | ||
| "@types/mysql": "^2.15.21", | ||
| "@types/node": "^17.0.16", | ||
| "@types/pg": "^8.6.4", | ||
| "json2csv": "^5.0.6", | ||
| "mysql": "^2.18.1", | ||
| "pg": "^8.4.2", | ||
| "pg-copy-streams": "^5.1.1", | ||
| "@types/mysql": "^2.15.15", | ||
| "@types/node": "^14.14.5", | ||
| "@types/pg": "^7.14.5" | ||
| "pg": "^8.7.3", | ||
| "pg-copy-streams": "^6.0.2" | ||
| }, | ||
| "devDependencies": { | ||
| "@types/tape": "^4.13.0", | ||
| "tape": "^5.0.1", | ||
| "typescript": "^4.0.5" | ||
| "@types/tape": "^4.13.2", | ||
| "tape": "^5.5.0", | ||
| "typescript": "4.5.5" | ||
| }, | ||
| "scripts": { | ||
| "build": "tsc", | ||
| "build": "tsc --incremental -p tsconfig.json", | ||
| "start": "node dist/src/Main.js", | ||
| "test": "node dist/test/Main.test.js" | ||
| }, | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,13 +18,14 @@ | |
| * | ||
| * @author Anatoly Khaytovich <[email protected]> | ||
| */ | ||
| import { PoolClient } from 'pg'; | ||
|
|
||
| import { log } from './FsOps'; | ||
| import Conversion from './Conversion'; | ||
| import DBAccess from './DBAccess'; | ||
| import DBAccessQueryResult from './DBAccessQueryResult'; | ||
| import DBVendors from './DBVendors'; | ||
| import IDBAccessQueryParams from './IDBAccessQueryParams'; | ||
| import { PoolClient } from 'pg'; | ||
|
|
||
| /** | ||
| * Decodes binary data from from textual representation in string. | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,7 @@ | |
| * @author Anatoly Khaytovich <[email protected]> | ||
| */ | ||
| import * as path from 'path'; | ||
|
|
||
| import Conversion from './Conversion'; | ||
| import DBAccess from './DBAccess'; | ||
| import DBAccessQueryResult from './DBAccessQueryResult'; | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,7 +18,7 @@ | |
| * | ||
| * @author Anatoly Khaytovich <[email protected]> | ||
| */ | ||
| import * as identityProcessor from './IdentityProcessor'; | ||
| import * as sequencesProcessor from './SequencesProcessor'; | ||
| import * as migrationStateManager from './MigrationStateManager'; | ||
| import processEnum from './EnumProcessor'; | ||
| import processNull from './NullProcessor'; | ||
|
|
@@ -66,13 +66,13 @@ export const processConstraintsPerTable = async ( | |
| migrateOnlyData: boolean | ||
| ): Promise<void> => { | ||
| if (migrateOnlyData) { | ||
| return identityProcessor.setSequenceValue(conversion, tableName); | ||
| return sequencesProcessor.setSequenceValue(conversion, tableName); | ||
| } | ||
|
|
||
| await processEnum(conversion, tableName); | ||
| await processNull(conversion, tableName); | ||
| await processDefault(conversion, tableName); | ||
| await identityProcessor.createIdentity(conversion, tableName); | ||
| await sequencesProcessor.createIdentity(conversion, tableName); | ||
| await processIndexAndKey(conversion, tableName); | ||
| await processComments(conversion, tableName); | ||
| }; | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,6 +18,8 @@ | |
| * | ||
| * @author Anatoly Khaytovich <[email protected]> | ||
| */ | ||
| import * as path from 'path'; | ||
|
|
||
| import { log, generateError } from './FsOps'; | ||
| import Conversion from './Conversion'; | ||
| import DBAccess from './DBAccess'; | ||
|
|
@@ -29,16 +31,17 @@ import { dataTransferred } from './ConsistencyEnforcer'; | |
| import IDBAccessQueryParams from './IDBAccessQueryParams'; | ||
| import * as extraConfigProcessor from './ExtraConfigProcessor'; | ||
| import { getDataPoolTableName } from './DataPoolManager'; | ||
| import * as path from 'path'; | ||
|
|
||
| import { PoolClient, QueryResult } from 'pg'; | ||
| import { PoolConnection } from 'mysql'; | ||
|
|
||
| const { from } = require('pg-copy-streams'); // No declaration file for module "pg-copy-streams". | ||
| const { Transform: Json2CsvTransform } = require('json2csv'); // No declaration file for module "json2csv". | ||
|
|
||
| process.on('message', async (signal: MessageToDataLoader) => { | ||
| const { config, chunk } = signal; | ||
| const conv: Conversion = new Conversion(config); | ||
| log(conv, `\t--[loadData] Loading the data into "${ conv._schema }"."${ chunk._tableName }" table...`); | ||
| log(conv, `\t--[NMIG loadData] Loading the data into "${ conv._schema }"."${ chunk._tableName }" table...`); | ||
|
|
||
| const isRecoveryMode: boolean = await dataTransferred(conv, chunk._id); | ||
|
|
||
|
|
||
Oops, something went wrong.