diff --git a/README.md b/README.md index 209e358b..44553a6e 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Or, if you have moved config folder out from Nmig's directory:
Note: "logs_directory" will be created during script execution.

VERSION

-

Current version is 5.4.0
+

Current version is 5.5.0

LICENSE

NMIG is available under "GNU GENERAL PUBLIC LICENSE" (v. 3)
diff --git a/config/index_types_map.json b/config/index_types_map.json new file mode 100644 index 00000000..88aac79f --- /dev/null +++ b/config/index_types_map.json @@ -0,0 +1,16 @@ +{ + "README" : [ + "This JSON document represents a correct index-types map between MySQL and PostgreSQL.", + "If you wish to customize (not recommended) this map - you can.", + "Map explanation:", + "Each key represents a MySQL index-type, and each value represents corresponding PostgreSQL index-type." + ], + + "BTREE": "BTREE", + + "HASH": "HASH", + + "SPATIAL": "GIST", + + "FULLTEXT": "GIN" +} diff --git a/package-lock.json b/package-lock.json index 35489b40..1657d032 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "nmig", - "version": "5.4.0", + "version": "5.5.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -13,14 +13,14 @@ } }, "@types/node": { - "version": "14.10.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.10.1.tgz", - "integrity": "sha512-aYNbO+FZ/3KGeQCEkNhHFRIzBOUgc7QvcVNKXbfnhDkSfwUv91JsQQa10rDgKSTSLkXZ1UIyPe4FJJNVgw1xWQ==" + "version": "14.14.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.5.tgz", + "integrity": "sha512-H5Wn24s/ZOukBmDn03nnGTp18A60ny9AmCwnEcgJiTgSGsCO7k+NWP7zjCCbhlcnVCoI+co52dUAt9GMhOSULw==" }, "@types/pg": { - "version": "7.14.4", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-7.14.4.tgz", - "integrity": "sha512-yCKVMCcFPZSFHGg+8qjY368uf3ruyDBPjxvOU2ZcGa/vRFo5Ti5Y6z6vl+2hxtwm9VMWUGb6TWkIk3cIV8C0Cw==", + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-7.14.5.tgz", + "integrity": "sha512-wqTKZmqkqXd1YiVRBT2poRrMIojwEi2bKTAAjUX6nEbzr98jc3cfR/7o7ZtubhH5xT7YJ6LRdRr1GZOgs8OUjg==", "requires": { "@types/node": "*", "@types/pg-types": "*" @@ -82,9 +82,9 @@ "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" }, "commander": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", - "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==" + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.0.tgz", + "integrity": "sha512-zP4jEKbe8SHzKJYQmq8Y9gYjtO/POJLgIdKgV7B9qNmABVFVc+ctqSX6iXh4mCpJfRBOabiZ2YKPg8ciDw6C+Q==" }, "concat-map": { "version": "0.0.1", @@ -98,22 +98,22 @@ "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "deep-equal": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.0.3.tgz", - "integrity": "sha512-Spqdl4H+ky45I9ByyJtXteOm9CaIrPmnIPmOhrkKGNYWeDgCvJ8jNYVCTjChxW4FqGuZnLHADc8EKRMX6+CgvA==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.0.4.tgz", + "integrity": "sha512-BUfaXrVoCfgkOQY/b09QdO9L3XNoF2XH0A3aY9IQwQL/ZjLOe8FQgCNVl1wiolhsFo8kFdO9zdPViCPbmaJA5w==", "dev": true, "requires": { - "es-abstract": "^1.17.5", + "es-abstract": "^1.18.0-next.1", "es-get-iterator": "^1.1.0", "is-arguments": "^1.0.4", "is-date-object": "^1.0.2", - "is-regex": "^1.0.5", + "is-regex": "^1.1.1", "isarray": "^2.0.5", - "object-is": "^1.1.2", + "object-is": "^1.1.3", "object-keys": "^1.1.1", - "object.assign": "^4.1.0", + "object.assign": "^4.1.1", "regexp.prototype.flags": "^1.3.0", - "side-channel": "^1.0.2", + "side-channel": "^1.0.3", "which-boxed-primitive": "^1.0.1", "which-collection": "^1.0.1", "which-typed-array": "^1.1.2" @@ -152,20 +152,21 @@ } }, "es-abstract": { - "version": "1.17.6", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", - "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "version": "1.18.0-next.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.1.tgz", + "integrity": "sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA==", "dev": true, "requires": { "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", "has": "^1.0.3", "has-symbols": "^1.0.1", - "is-callable": "^1.2.0", - "is-regex": "^1.1.0", - "object-inspect": "^1.7.0", + "is-callable": "^1.2.2", + "is-negative-zero": "^2.0.0", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", "object-keys": "^1.1.1", - "object.assign": "^4.1.0", + "object.assign": "^4.1.1", "string.prototype.trimend": "^1.0.1", "string.prototype.trimstart": "^1.0.1" } @@ -185,6 +186,25 @@ "isarray": "^2.0.5" }, "dependencies": { + "es-abstract": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.7.tgz", + "integrity": "sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g==", + "dev": true, + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.2", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.1", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, "isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", @@ -294,11 +314,20 @@ "dev": true }, "is-callable": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", - "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz", + "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==", "dev": true }, + "is-core-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.0.0.tgz", + "integrity": "sha512-jq1AH6C8MuteOoBPwkxHafmByhL9j5q4OaPGdbuD+ZtQJVzH+i6E3BJDQcBA09k57i2Hh2yQbEG8yObZ0jdlWw==", + "dev": true, + "requires": { + "has": "^1.0.3" + } + }, "is-date-object": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", @@ -311,6 +340,12 @@ "integrity": "sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw==", "dev": true }, + "is-negative-zero": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.0.tgz", + "integrity": "sha1-lVOxIbD6wohp2p7UWeIMdUN4hGE=", + "dev": true + }, "is-number-object": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.4.tgz", @@ -318,9 +353,9 @@ "dev": true }, "is-regex": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", - "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", + "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==", "dev": true, "requires": { "has-symbols": "^1.0.1" @@ -357,6 +392,27 @@ "es-abstract": "^1.17.4", "foreach": "^2.0.5", "has-symbols": "^1.0.1" + }, + "dependencies": { + "es-abstract": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.7.tgz", + "integrity": "sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g==", + "dev": true, + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.2", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.1", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + } } }, "is-weakmap": { @@ -377,11 +433,11 @@ "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "json2csv": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json2csv/-/json2csv-5.0.1.tgz", - "integrity": "sha512-QFMifUX1y8W2tKi2TwZpnzf2rHdZvzdmgZUMEMDF46F90f4a9mUeWfx/qg4kzXSZYJYc3cWA5O+eLXk5lj9g8g==", + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/json2csv/-/json2csv-5.0.3.tgz", + "integrity": "sha512-e3gEZU/4fp8CVQMHlwT77RayAR7nylCzCYN7jTIbPTEqk0oTaE8GTcBudLgXrHt4ltOs9SAsbveMJT0YK/QUSg==", "requires": { - "commander": "^5.0.0", + "commander": "^6.1.0", "jsonparse": "^1.3.1", "lodash.get": "^4.4.2" } @@ -429,13 +485,13 @@ "dev": true }, "object-is": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.2.tgz", - "integrity": "sha512-5lHCz+0uufF6wZ7CRFWJN3hp8Jqblpgve06U5CMQ3f//6iDjPr2PEo9MWCjEssDsa+UZEL4PkFpr+BMop6aKzQ==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.3.tgz", + "integrity": "sha512-teyqLvFWzLkq5B9ki8FVWA902UER2qkxmdA4nLf+wjOLAWgxzCWZNCxpDq9MvE8MmhWNr+I8w3BN49Vx36Y6Xg==", "dev": true, "requires": { "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" + "es-abstract": "^1.18.0-next.1" } }, "object-keys": { @@ -445,17 +501,22 @@ "dev": true }, "object.assign": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", - "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.1.tgz", + "integrity": "sha512-VT/cxmx5yaoHSOTSyrCygIDFco+RsibY2NM0a4RdEeY/4KgqezwFtK1yr3U67xYhqJSlASm2pKhLVzPj2lr4bA==", "dev": true, "requires": { - "define-properties": "^1.1.2", - "function-bind": "^1.1.1", - "has-symbols": "^1.0.0", - "object-keys": "^1.0.11" + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.0", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" } }, + "obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==" + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -483,29 +544,31 @@ "dev": true }, "pg": { - "version": "8.3.3", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.3.3.tgz", - "integrity": "sha512-wmUyoQM/Xzmo62wgOdQAn5tl7u+IA1ZYK7qbuppi+3E+Gj4hlUxVHjInulieWrd0SfHi/ADriTb5ILJ/lsJrSg==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.4.2.tgz", + "integrity": "sha512-E9FlUrrc7w3+sbRmL1CSw99vifACzB2TjhMM9J5w9D1LIg+6un0jKkpHS1EQf2CWhKhec2bhrBLVMmUBDbjPRQ==", "requires": { "buffer-writer": "2.0.0", "packet-reader": "1.0.0", - "pg-connection-string": "^2.3.0", - "pg-pool": "^3.2.1", - "pg-protocol": "^1.2.5", + "pg-connection-string": "^2.4.0", + "pg-pool": "^3.2.2", + "pg-protocol": "^1.3.0", "pg-types": "^2.1.0", - "pgpass": "1.x", - "semver": "4.3.2" + "pgpass": "1.x" } }, "pg-connection-string": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.3.0.tgz", - "integrity": "sha512-ukMTJXLI7/hZIwTW7hGMZJ0Lj0S2XQBCJ4Shv4y1zgQ/vqVea+FLhzywvPj0ujSuofu+yA4MYHGZPTsgjBgJ+w==" + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.4.0.tgz", + "integrity": "sha512-3iBXuv7XKvxeMrIgym7njT+HlZkwZqqGX4Bu9cci8xHZNT+Um1gWKqCsAzcC0d95rcKMU5WBg6YRUcHyV0HZKQ==" }, "pg-copy-streams": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-2.2.2.tgz", - "integrity": "sha512-mjSqs6hrsRhBojCuY2hxyg48B+3th5ARBjMxBCEisIqBvdRD0g5ETdbts20TzrOfha8ueJQOmQCJCprtczJtGQ==" + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-5.1.1.tgz", + "integrity": "sha512-ieW6JuiIo/4WQ7n+Wevr9zYvpM1AwUs6EwNCCA0VgKZ6ZQ7Y9k3IW00vqc6svX9FtENhbaTbLN7MxekraCrbfg==", + "requires": { + "obuf": "^1.1.2" + } }, "pg-int8": { "version": "1.0.1", @@ -513,14 +576,14 @@ "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" }, "pg-pool": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.2.1.tgz", - "integrity": "sha512-BQDPWUeKenVrMMDN9opfns/kZo4lxmSWhIqo+cSAF7+lfi9ZclQbr9vfnlNaPr8wYF3UYjm5X0yPAhbcgqNOdA==" + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.2.2.tgz", + "integrity": "sha512-ORJoFxAlmmros8igi608iVEbQNNZlp89diFVx6yV5v+ehmpMY9sK6QgpmgoXbmkNaBAx8cOOZh9g80kJv1ooyA==" }, "pg-protocol": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.2.5.tgz", - "integrity": "sha512-1uYCckkuTfzz/FCefvavRywkowa6M5FohNMF5OjKrqo9PSR8gYc8poVmwwYQaBxhmQdBjhtP514eXy9/Us2xKg==" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.3.0.tgz", + "integrity": "sha512-64/bYByMrhWULUaCd+6/72c9PMWhiVFs3EVxl9Ct6a3v/U8+rKgqP2w+kKg/BIGgMJyB+Bk/eNivT32Al+Jghw==" }, "pg-types": { "version": "2.2.0", @@ -535,11 +598,11 @@ } }, "pgpass": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", - "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.4.tgz", + "integrity": "sha512-YmuA56alyBq7M59vxVBfPJrGSozru8QAdoNlWuW3cz8l+UX3cWge0vTvjKhsSHSJpo3Bom8/Mm6hf0TR5GY0+w==", "requires": { - "split": "^1.0.0" + "split2": "^3.1.1" } }, "postgres-array": { @@ -592,14 +655,36 @@ "requires": { "define-properties": "^1.1.3", "es-abstract": "^1.17.0-next.1" + }, + "dependencies": { + "es-abstract": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.7.tgz", + "integrity": "sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g==", + "dev": true, + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.2", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.1", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + } } }, "resolve": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz", - "integrity": "sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==", + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.18.1.tgz", + "integrity": "sha512-lDfCPaMKfOJXjy0dPayzPdF1phampNWr3qFCjAu+rw/qbQmr5jWH5xN2hwh9QKfw9E5v4hwV7A+jrCmL8yjjqA==", "dev": true, "requires": { + "is-core-module": "^2.0.0", "path-parse": "^1.0.6" } }, @@ -617,27 +702,34 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, - "semver": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", - "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" - }, "side-channel": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.2.tgz", - "integrity": "sha512-7rL9YlPHg7Ancea1S96Pa8/QWb4BtXL/TZvS6B8XFetGBeuhAsfmUspK6DokBeZ64+Kj9TCNRD/30pVz1BvQNA==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.3.tgz", + "integrity": "sha512-A6+ByhlLkksFoUepsGxfj5x1gTSrs+OydsRptUxeNCabQpCFUvcwIczgOigI8vhY/OJCnPnyE9rGiwgvr9cS1g==", "dev": true, "requires": { - "es-abstract": "^1.17.0-next.1", - "object-inspect": "^1.7.0" + "es-abstract": "^1.18.0-next.0", + "object-inspect": "^1.8.0" } }, - "split": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", - "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "split2": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", + "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", "requires": { - "through": "2" + "readable-stream": "^3.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } } }, "sqlstring": { @@ -646,34 +738,33 @@ "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=" }, "string.prototype.trim": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.1.tgz", - "integrity": "sha512-MjGFEeqixw47dAMFMtgUro/I0+wNqZB5GKXGt1fFr24u3TzDXCPu7J9Buppzoe3r/LqkSDLDDJzE15RGWDGAVw==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.2.tgz", + "integrity": "sha512-b5yrbl3BXIjHau9Prk7U0RRYcUYdN4wGSVaqoBQS50CCE3KBuYU0TYRNPFCP7aVoNMX87HKThdMRVIP3giclKg==", "dev": true, "requires": { "define-properties": "^1.1.3", - "es-abstract": "^1.17.0-next.1", - "function-bind": "^1.1.1" + "es-abstract": "^1.18.0-next.0" } }, "string.prototype.trimend": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", - "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.2.tgz", + "integrity": "sha512-8oAG/hi14Z4nOVP0z6mdiVZ/wqjDtWSLygMigTzAb+7aPEDTleeFf+WrF+alzecxIRkckkJVn+dTlwzJXORATw==", "dev": true, "requires": { "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" + "es-abstract": "^1.18.0-next.1" } }, "string.prototype.trimstart": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", - "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.2.tgz", + "integrity": "sha512-7F6CdBTl5zyu30BJFdzSTlSlLPwODC23Od+iLoVH8X6+3fvDPPuBVVj9iaB1GOsSTSIgVfsfm27R2FGrAPznWg==", "dev": true, "requires": { "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" + "es-abstract": "^1.18.0-next.1" } }, "string_decoder": { @@ -712,12 +803,13 @@ "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", + "dev": true }, "typescript": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.0.2.tgz", - "integrity": "sha512-e4ERvRV2wb+rRZ/IQeb3jm2VxBsirQLpQhdxplZ2MEzGvDkkMmPglecnNDfSUBivMjP93vRbngYYDQqQ/78bcQ==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.0.5.tgz", + "integrity": "sha512-ywmr/VrTVCmNTJ6iV2LwIrfG1P+lv6luD8sUJs+2eI9NLGigaN+nUQc13iHqisq7bra9lnmUSYqbJvegraBOPQ==", "dev": true }, "util-deprecate": { @@ -762,6 +854,27 @@ "function-bind": "^1.1.1", "has-symbols": "^1.0.1", "is-typed-array": "^1.1.3" + }, + "dependencies": { + "es-abstract": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.7.tgz", + "integrity": "sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g==", + "dev": true, + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.2", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.1", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + } } }, "wrappy": { diff --git a/package.json b/package.json index 3c8bf0da..c594a7a1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "nmig", - "version": "5.4.0", + "version": "5.5.0", "description": "The database migration app", "author": "Anatoly Khaytovich", "license": "GPL-3.0", @@ -12,18 +12,18 @@ "node": ">=10.0.0" }, "dependencies": { - "json2csv": "^5.0.1", + "json2csv": "^5.0.3", "mysql": "^2.18.1", - "pg": "^8.3.3", - "pg-copy-streams": "^2.2.2", + "pg": "^8.4.2", + "pg-copy-streams": "^5.1.1", "@types/mysql": "^2.15.15", - "@types/node": "^14.10.1", - "@types/pg": "^7.14.4" + "@types/node": "^14.14.5", + "@types/pg": "^7.14.5" }, "devDependencies": { "@types/tape": "^4.13.0", "tape": "^5.0.1", - "typescript": "^4.0.2" + "typescript": "^4.0.5" }, "scripts": { "build": "tsc", diff --git a/src/BootProcessor.ts b/src/BootProcessor.ts index bc10ff6e..cf50e4d7 100644 --- a/src/BootProcessor.ts +++ b/src/BootProcessor.ts @@ -26,6 +26,7 @@ import DBVendors from './DBVendors'; import IDBAccessQueryParams from './IDBAccessQueryParams'; import IConfAndLogsPaths from './IConfAndLogsPaths'; import { getStateLogsTableName } from './MigrationStateManager'; +import { generateError, log } from './FsOps'; /** * Checks correctness of connection details of both MySQL and PostgreSQL. @@ -67,61 +68,36 @@ export const getLogo = (): string => { /** * Boots the migration. */ -export const boot = (conversion: Conversion): Promise => { - return new Promise(async resolve => { - const connectionErrorMessage = await checkConnection(conversion); - const logo: string = getLogo(); +export const boot = async (conversion: Conversion): Promise => { + const connectionErrorMessage = await checkConnection(conversion); + const logo: string = getLogo(); + const logTitle: string = 'BootProcessor::boot'; - if (connectionErrorMessage) { - console.log(`${ logo } \n ${ connectionErrorMessage }`); - process.exit(1); - } + if (connectionErrorMessage) { + await generateError(conversion, `\t--[${ logTitle }]\n ${ logo } \n ${ connectionErrorMessage }`); + process.exit(1); + } - const sql: string = `SELECT EXISTS(SELECT 1 FROM information_schema.tables` - + ` WHERE table_schema = '${ conversion._schema }' AND table_name = '${ getStateLogsTableName(conversion, true) }');`; + const sql: string = `SELECT EXISTS(SELECT 1 FROM information_schema.tables` + + ` WHERE table_schema = '${ conversion._schema }' AND table_name = '${ getStateLogsTableName(conversion, true) }');`; - const params: IDBAccessQueryParams = { - conversion: conversion, - caller: 'BootProcessor::boot', - sql: sql, - vendor: DBVendors.PG, - processExitOnError: true, - shouldReturnClient: false - }; - - const result: DBAccessQueryResult = await DBAccess.query(params); - const isExists: boolean = !!result.data.rows[0].exists; - const message: string = `${ (isExists - ? '\n\t--[boot] NMIG is ready to restart after some failure.\n\t--[boot] Consider checking log files at the end of migration.' - : '\n\t--[boot] NMIG is ready to start.') } \n\t--[boot] Proceed? [Y/n]`; - - console.log(logo + message); - - const _getUserInput = (input: string): void => { - const trimedInput: string = input.trim(); - - if (trimedInput === 'n' || trimedInput === 'N') { - console.log('\t--[boot] Migration aborted.\n'); - process.exit(0); - } - - if (trimedInput === 'y' || trimedInput === 'Y') { - process.stdin.removeListener('data', _getUserInput); - conversion._timeBegin = new Date(); - return resolve(conversion); - } - - const hint: string = `\t--[boot] Unexpected input ${ trimedInput }\n` - + `\t--[boot] Expected input is upper case Y\n\t--[boot] or lower case n\n${message}`; + const params: IDBAccessQueryParams = { + conversion: conversion, + caller: 'BootProcessor::boot', + sql: sql, + vendor: DBVendors.PG, + processExitOnError: true, + shouldReturnClient: false + }; - console.log(hint); - }; + const result: DBAccessQueryResult = await DBAccess.query(params); + const isExists: boolean = !!result.data.rows[0].exists; + const message: string = `${ (isExists + ? '\n\t--[boot] NMIG is restarting after some failure.\n\t--[boot] Consider checking log files at the end of migration.\n' + : '\n\t--[boot] NMIG is starting.') } \n`; - process.stdin - .resume() - .setEncoding(conversion._encoding) - .on('data', _getUserInput); - }); + log(conversion, `\t--[${ logTitle }] ${ logo }${ message }`); + return conversion; }; /** diff --git a/src/Conversion.ts b/src/Conversion.ts index 9a4accd9..41ca7891 100644 --- a/src/Conversion.ts +++ b/src/Conversion.ts @@ -85,6 +85,11 @@ export default class Conversion { */ public readonly _dataTypesMapAddr: string; + /** + * A path to the index types map. + */ + public readonly _indexTypesMapAddr: string; + /** * A path to the "errors-only.log" file. */ @@ -108,7 +113,7 @@ export default class Conversion { /** * The timestamp, at which the migration began. */ - public _timeBegin: Date | null; + public _timeBegin: Date; /** * Current version of source (MySQL) db. @@ -185,6 +190,11 @@ export default class Conversion { */ public _dataTypesMap: any; + /** + * The index types map. + */ + public _indexTypesMap: any; + /** * Buffer level when stream.write() starts returning false. * This number is a number of JavaScript objects. @@ -200,12 +210,13 @@ export default class Conversion { this._targetConString = this._config.target; this._logsDirPath = this._config.logsDirPath; this._dataTypesMapAddr = this._config.dataTypesMapAddr; + this._indexTypesMapAddr = this._config.indexTypesMapAddr; this._allLogsPath = path.join(this._logsDirPath, 'all.log'); this._errorLogsPath = path.join(this._logsDirPath, 'errors-only.log'); this._notCreatedViewsPath = path.join(this._logsDirPath, 'not_created_views'); this._excludeTables = this._config.exclude_tables === undefined ? [] : this._config.exclude_tables; this._includeTables = this._config.include_tables === undefined ? [] : this._config.include_tables; - this._timeBegin = null; + this._timeBegin = new Date(); this._encoding = this._config.encoding === undefined ? 'utf8' : this._config.encoding; this._0777 = '0777'; this._mysqlVersion = '5.6.21'; // Simply a default value. diff --git a/src/DataLoader.ts b/src/DataLoader.ts index 6a99bfbb..32ad26ee 100644 --- a/src/DataLoader.ts +++ b/src/DataLoader.ts @@ -165,10 +165,10 @@ const getCopyStream = ( const copyStream: any = client.query(from(sqlCopy)); copyStream - .on('end', async () => { + .on('finish', async () => { // COPY FROM STDIN does not return the number of rows inserted. // But the transactional behavior still applies, meaning no records inserted if at least one failed. - // That is why in case of 'on end' the rowsCnt value is actually the number of records inserted. + // That is why in case of 'on finish' the rowsCnt value is actually the number of records inserted. processSend(new MessageToMaster(tableName, rowsCnt)); await deleteChunk(conv, dataPoolId, client); }) diff --git a/src/FsOps.ts b/src/FsOps.ts index 70363148..d8c566bb 100644 --- a/src/FsOps.ts +++ b/src/FsOps.ts @@ -84,48 +84,57 @@ export const log = (conversion: Conversion, log: string | NodeJS.ErrnoException, }; /** - * Reads the configuration file. + * Reads and parses JOSN file under given path. */ -export const readConfig = (confPath: string, logsPath: string, configFileName: string = 'config.json'): Promise => { +const readAndParseJsonFile = (pathToFile: string): Promise => { return new Promise(resolve => { - const pathToConfig = path.join(confPath, configFileName); - - fs.readFile(pathToConfig, (error: ErrnoException | null, data: Buffer) => { + fs.readFile(pathToFile, (error: ErrnoException | null, data: Buffer) => { if (error) { - console.log(`\n\t--Cannot run migration\nCannot read configuration info from ${ pathToConfig }`); + console.log(`\n\t--Cannot run migration\nCannot read configuration info from ${ pathToFile }`); process.exit(1); } const config: any = JSON.parse(data.toString()); - config.logsDirPath = path.join(logsPath, 'logs_directory'); - config.dataTypesMapAddr = path.join(confPath, 'data_types_map.json'); resolve(config); }); }); }; /** - * Reads the extra configuration file, if necessary. + * Reads the configuration file. */ -export const readExtraConfig = (config: any, confPath: string, extraConfigFileName: string = 'extra_config.json'): Promise => { - return new Promise(resolve => { - if (config.enable_extra_config !== true) { - config.extraConfig = null; - return resolve(config); - } - - const pathToExtraConfig = path.join(confPath, extraConfigFileName); +export const readConfig = async (confPath: string, logsPath: string, configFileName: string = 'config.json'): Promise => { + const pathToConfig = path.join(confPath, configFileName); + const config: any = await readAndParseJsonFile(pathToConfig); + config.logsDirPath = path.join(logsPath, 'logs_directory'); + config.dataTypesMapAddr = path.join(confPath, 'data_types_map.json'); + config.indexTypesMapAddr = path.join(confPath, 'index_types_map.json'); + return config; +}; - fs.readFile(pathToExtraConfig, (error: ErrnoException | null, data: Buffer) => { - if (error) { - console.log(`\n\t--Cannot run migration\nCannot read configuration info from ${ pathToExtraConfig }`); - process.exit(1); - } +/** + * Reads the extra configuration file, if necessary. + */ +export const readExtraConfig = async (config: any, confPath: string, extraConfigFileName: string = 'extra_config.json'): Promise => { + if (config.enable_extra_config !== true) { + config.extraConfig = null; + return config; + } + + const pathToExtraConfig = path.join(confPath, extraConfigFileName); + config.extraConfig = await readAndParseJsonFile(pathToExtraConfig); + return config; +}; - config.extraConfig = JSON.parse(data.toString()); - resolve(config); - }); - }); +/** + * Reads both "./config/data_types_map.json" and "./config/index_types_map.json" and converts its json content to js object. + */ +export const readDataAndIndexTypesMap = async (conversion: Conversion): Promise => { + const logTitle: string = 'FsOps::readDataAndIndexTypesMap'; + conversion._dataTypesMap = await readAndParseJsonFile(conversion._dataTypesMapAddr); + conversion._indexTypesMap = await readAndParseJsonFile(conversion._indexTypesMapAddr); + log(conversion, `\t--[${ logTitle }] Data and Index Types Maps are loaded...`); + return conversion; }; /** @@ -166,23 +175,3 @@ const createDirectory = (conversion: Conversion, directoryPath: string, logTitle }); }); }; - -/** - * Reads "./config/data_types_map.json" and converts its json content to js object. - */ -export const readDataTypesMap = (conversion: Conversion): Promise => { - return new Promise(resolve => { - fs.readFile(conversion._dataTypesMapAddr, (error: ErrnoException | null, data: Buffer) => { - const logTitle: string = 'FsOps::readDataTypesMap'; - - if (error) { - console.log(`\t--[${ logTitle }] Cannot read "DataTypesMap" from ${conversion._dataTypesMapAddr}`); - process.exit(1); - } - - conversion._dataTypesMap = JSON.parse(data.toString()); - console.log(`\t--[${ logTitle }] Data Types Map is loaded...`); - resolve(conversion); - }); - }); -}; diff --git a/src/IndexAndKeyProcessor.ts b/src/IndexAndKeyProcessor.ts index 297692ef..c95aa3c6 100644 --- a/src/IndexAndKeyProcessor.ts +++ b/src/IndexAndKeyProcessor.ts @@ -26,6 +26,13 @@ import DBAccessQueryResult from './DBAccessQueryResult'; import IDBAccessQueryParams from './IDBAccessQueryParams'; import * as extraConfigProcessor from './ExtraConfigProcessor'; +/** + * Returns PostgreSQL index type, that correlates to given MySQL index type. + */ +const getIndexType = (conversion: Conversion, indexType: string): string => { + return indexType in conversion._indexTypesMap ? conversion._indexTypesMap[indexType] : 'BTREE'; +}; + /** * Creates primary key and indices. */ @@ -62,7 +69,7 @@ export default async (conversion: Conversion, tableName: string): Promise objPgIndices[index.Key_name] = { is_unique: index.Non_unique === 0, column_name: [`"${ pgColumnName }"`], - Index_type: ` USING ${ index.Index_type === 'SPATIAL' ? 'GIST' : index.Index_type }` + index_type: ` USING ${ getIndexType(conversion, index.Index_type) }`, }; }); @@ -79,7 +86,7 @@ export default async (conversion: Conversion, tableName: string): Promise indexType = 'index'; sqlAddIndex = `CREATE ${ (objPgIndices[index].is_unique ? 'UNIQUE ' : '') }INDEX "${ conversion._schema }_${ tableName }_${ columnName }_idx" ON "${ conversion._schema }"."${ tableName }" - ${ objPgIndices[index].Index_type } (${ objPgIndices[index].column_name.join(',') });`; + ${ objPgIndices[index].index_type } (${ objPgIndices[index].column_name.join(',') });`; } params.vendor = DBVendors.PG; diff --git a/src/Main.ts b/src/Main.ts index c104e75a..7f4b9630 100644 --- a/src/Main.ts +++ b/src/Main.ts @@ -30,16 +30,16 @@ import { processConstraints } from './ConstraintsProcessor'; import { getConfAndLogsPaths, boot } from './BootProcessor'; import { createStateLogsTable, dropStateLogsTable } from './MigrationStateManager'; import { createDataPoolTable, readDataPool } from './DataPoolManager'; -import { readConfig, readExtraConfig, createLogsDirectory, readDataTypesMap } from './FsOps'; +import { readConfig, readExtraConfig, createLogsDirectory, readDataAndIndexTypesMap } from './FsOps'; const { confPath, logsPath } = getConfAndLogsPaths(); readConfig(confPath, logsPath) .then(config => readExtraConfig(config, confPath)) .then(Conversion.initializeConversion) - .then(boot) - .then(readDataTypesMap) .then(createLogsDirectory) + .then(readDataAndIndexTypesMap) + .then(boot) .then(createSchema) .then(createStateLogsTable) .then(createDataPoolTable) diff --git a/test/TestModules/TestSchemaProcessor.ts b/test/TestModules/TestSchemaProcessor.ts index 2cf6cfc4..1d754c57 100644 --- a/test/TestModules/TestSchemaProcessor.ts +++ b/test/TestModules/TestSchemaProcessor.ts @@ -36,7 +36,7 @@ import { processConstraints } from '../../src/ConstraintsProcessor'; import { createStateLogsTable, dropStateLogsTable } from '../../src/MigrationStateManager'; import { createDataPoolTable, readDataPool } from '../../src/DataPoolManager'; import { checkConnection, getLogo, getConfAndLogsPaths } from '../../src/BootProcessor'; -import { createLogsDirectory, generateError, log, readConfig, readDataTypesMap, readExtraConfig } from '../../src/FsOps'; +import { createLogsDirectory, generateError, log, readConfig, readDataAndIndexTypesMap, readExtraConfig } from '../../src/FsOps'; import ErrnoException = NodeJS.ErrnoException; export default class TestSchemaProcessor { @@ -294,7 +294,7 @@ export default class TestSchemaProcessor { .then(this._updateMySqlConnection.bind(this)) .then(this._loadTestSchema.bind(this)) .then(this._loadTestData.bind(this)) - .then(readDataTypesMap) + .then(readDataAndIndexTypesMap) .then(createLogsDirectory) .then(createSchema) .then(createStateLogsTable) @@ -307,7 +307,6 @@ export default class TestSchemaProcessor { .then(dropDataPoolTable) .then(dropStateLogsTable) .then(DBAccess.closeConnectionPools) - .then(generateReport) - .catch(error => console.log(error)); + .then(generateReport); } }