Add optional param hex to getTransaction and getBlock. (#5845)

* allow `hexToNumber` to handle numbers smaller than `MIN_SAFE_INTEGER` by adding `bigIntOnOverflow` optional param and default it to `false` * add optional `hexFormat` param to `getTransaction` and `getBlock` * update test cases for `utils.toNumber` * update CHANGELOG.md * updates `hexToNumber` documentation
web3 · Feb 23, 2023 · e4b25bf · e4b25bf
1 parent 8621030
commit e4b25bf
Show file tree

Hide file tree

Showing 10 changed files with 94 additions and 55 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -646,3 +646,9 @@ Released with 1.0.0-beta.37 code base.
 
   - Fixed skipped ws-ganache tests (#5759)
   - Fixed "provider started to reconnect error" in web3-provider-ws (#5820)
+  - Fixed Error: Number can only safely store up to 53 bits (#5845)
+
+### Changed
+
+  - Add optional `hexFormat` param to `getTransaction` and `getBlock` that accepts the value `'hex'` (#5845)
+  - `utils.toNumber` and `utils.hexToNumber` can now return the large usafe numbers as `BigInt`, if `true` was passed to a new optional parameter called `bigIntOnOverflow` (#5845)
diff --git a/docs/web3-utils.rst b/docs/web3-utils.rst
@@ -772,12 +772,13 @@ hexToNumber
 
 .. code-block:: javascript
 
-    web3.utils.hexToNumber(hex)
+    web3.utils.hexToNumber(hex) // if it is larger than 53 bit, it will throw an error
+    web3.utils.hexToNumber(hex, true) // if it is larger than 53 bit, it will return the value as BigInt
     web3.utils.toDecimal(hex) // ALIAS, deprecated
 
-Returns the number representation of a given HEX value.
+Returns the number representation of a given HEX value. And only if the second parameter is passed as `true` and the number is very big (unsafe number), it will return the value as a `BigInt`.
 
-.. note:: This is not useful for big numbers, rather use :ref:`utils.toBN <utils-tobn>` instead.
+.. note:: To handle for big numbers, either use :ref:`utils.toBN <utils-tobn>` to return as `BN`. Or, pass `true` to the second parameter to return the value as `BigInt`, in case of an overflow.
 
 ----------
 Parameters

diff --git a/packages/web3-core-helpers/src/formatters.js b/packages/web3-core-helpers/src/formatters.js
@@ -235,21 +235,23 @@ var inputSignFormatter = function (data) {
  * @param {Object} tx
  * @returns {Object}
  */
-var outputTransactionFormatter = function (tx) {
-    if (tx.blockNumber !== null)
-        tx.blockNumber = utils.hexToNumber(tx.blockNumber);
-    if (tx.transactionIndex !== null)
-        tx.transactionIndex = utils.hexToNumber(tx.transactionIndex);
-    tx.nonce = utils.hexToNumber(tx.nonce);
-    tx.gas = utils.hexToNumber(tx.gas);
+var outputTransactionFormatter = function (tx, hexFormat) {
+    if (!hexFormat) {
+        if (tx.blockNumber !== null)
+            tx.blockNumber = utils.hexToNumber(tx.blockNumber);
+        if (tx.transactionIndex !== null)
+            tx.transactionIndex = utils.hexToNumber(tx.transactionIndex);
+        tx.nonce = utils.hexToNumber(tx.nonce);
+        tx.gas = utils.hexToNumber(tx.gas);
+        if (tx.type)
+            tx.type = utils.hexToNumber(tx.type);
+    }
     if (tx.gasPrice)
         tx.gasPrice = outputBigNumberFormatter(tx.gasPrice);
     if (tx.maxFeePerGas)
         tx.maxFeePerGas = outputBigNumberFormatter(tx.maxFeePerGas);
     if (tx.maxPriorityFeePerGas)
         tx.maxPriorityFeePerGas = outputBigNumberFormatter(tx.maxPriorityFeePerGas);
-    if (tx.type)
-        tx.type = utils.hexToNumber(tx.type);
     tx.value = outputBigNumberFormatter(tx.value);
 
     if (tx.to && utils.isAddress(tx.to)) { // tx.to could be `0x0` or `null` while contract creation
@@ -310,15 +312,17 @@ var outputTransactionReceiptFormatter = function (receipt) {
  * @param {Object} block
  * @returns {Object}
  */
-var outputBlockFormatter = function (block) {
-
-    // transform to number
-    block.gasLimit = utils.hexToNumber(block.gasLimit);
-    block.gasUsed = utils.hexToNumber(block.gasUsed);
-    block.size = utils.hexToNumber(block.size);
-    block.timestamp = utils.hexToNumber(block.timestamp);
-    if (block.number !== null)
-        block.number = utils.hexToNumber(block.number);
+var outputBlockFormatter = function (block, hexFormat) {
+
+    if (!hexFormat) {
+        // transform to number
+        block.gasLimit = utils.hexToNumber(block.gasLimit);
+        block.gasUsed = utils.hexToNumber(block.gasUsed);
+        block.size = utils.hexToNumber(block.size);
+        block.timestamp = utils.hexToNumber(block.timestamp);
+        if (block.number !== null)
+            block.number = utils.hexToNumber(block.number);
+    }
 
     if (block.difficulty)
         block.difficulty = outputBigNumberFormatter(block.difficulty);
@@ -328,7 +332,7 @@ var outputBlockFormatter = function (block) {
     if (Array.isArray(block.transactions)) {
         block.transactions.forEach(function (item) {
             if (!(typeof item === 'string'))
-                return outputTransactionFormatter(item);
+                return outputTransactionFormatter(item, hexFormat);
         });
     }
 

diff --git a/packages/web3-core-helpers/types/index.d.ts b/packages/web3-core-helpers/types/index.d.ts
@@ -34,7 +34,7 @@ export class formatters {
 
     static inputBlockNumberFormatter(blockNumber: string | number): string | number;
 
-    static outputBlockFormatter(block: any): any; // TODO: Create Block interface
+    static outputBlockFormatter(block: any, hexFormat?: boolean): any; // TODO: Create Block interface
 
     static txInputFormatter(txObject: any): any;
 

diff --git a/packages/web3-core-method/src/index.js b/packages/web3-core-method/src/index.js
@@ -164,10 +164,10 @@ Method.prototype.formatOutput = function (result) {
 
     if (Array.isArray(result)) {
         return result.map(function (res) {
-            return _this.outputFormatter && res ? _this.outputFormatter(res) : res;
+            return _this.outputFormatter && res ? _this.outputFormatter(res, this?.hexFormat) : res;
         });
     } else {
-        return this.outputFormatter && result ? this.outputFormatter(result) : result;
+        return this.outputFormatter && result ? this.outputFormatter(result, this?.hexFormat) : result;
     }
 };
 
@@ -637,7 +637,9 @@ Method.prototype.buildCall = function () {
             payload = method.toPayload(args);
 
         method.hexFormat = false;
-        if(method.call === 'eth_getTransactionReceipt'){
+        if (method.call === 'eth_getTransactionReceipt'
+            || method.call === 'eth_getTransactionByHash'
+            || method.name === 'getBlock') {
             method.hexFormat = (payload.params.length  < args.length && args[args.length - 1] === 'hex')
         }
         // CALLBACK function

diff --git a/packages/web3-utils/src/utils.js b/packages/web3-utils/src/utils.js
@@ -218,13 +218,15 @@ var hexToUtf8 = function(hex) {
 
 
 /**
- * Converts value to it's number representation
+ * Converts value to it's number representation.
+ * However, if the value is larger than the maximum safe integer, returns the value as a string.
  *
  * @method hexToNumber
  * @param {String|Number|BN} value
- * @return {String}
+ * @param {Boolean} bigIntOnOverflow - if true, return the hex value in case of overflow
+ * @return {Number|String}
  */
-var hexToNumber = function (value) {
+var hexToNumber = function (value, bigIntOnOverflow = false) {
     if (!value) {
         return value;
     }
@@ -233,7 +235,11 @@ var hexToNumber = function (value) {
         throw new Error('Given value "'+value+'" is not a valid hex string.');
     }
 
-    return toBN(value).toNumber();
+    const n = toBN(value);
+    if (bigIntOnOverflow && (n > Number.MAX_SAFE_INTEGER || n < Number.MIN_SAFE_INTEGER)) {
+        return BigInt(n);
+    }
+    return n.toNumber();
 };
 
 /**
@@ -528,10 +534,11 @@ var sha3Raw = function(value) {
  *
  * @method toNumber
  * @param {String|Number|BN} value
- * @return {Number}
+ * @param {Boolean} bigIntOnOverflow - if true, return the hex value in case of overflow
+ * @return {Number|String}
  */
-var toNumber = function(value) {
-    return typeof value === 'number' ? value : hexToNumber(toHex(value));
+var toNumber = function (value, bigIntOnOverflow = false) {
+    return typeof value === 'number' ? value : hexToNumber(toHex(value), bigIntOnOverflow);
 }
 
 // 1.x currently accepts 0x... strings, bn.js after update doesn't. it would be a breaking change

diff --git a/packages/web3-utils/types/index.d.ts b/packages/web3-utils/types/index.d.ts
@@ -86,7 +86,7 @@ export function fromDecimal(value: string | number): string;
 export function fromUtf8(string: string): string;
 export function fromWei(value: string | BN, unit?: Unit): string;
 export function hexToBytes(hex: Hex): number[];
-export function hexToNumber(hex: Hex): number;
+export function hexToNumber(hex: Hex,bigIntOnOverflow?: boolean): number | string;
 export function hexToNumberString(hex: Hex): string;
 export function hexToString(hex: Hex): string;
 export function hexToUtf8(string: string): string;
@@ -122,7 +122,7 @@ export function testAddress(bloom: string, address: string): boolean;
 export function testTopic(bloom: string, topic: string): boolean;
 export function getSignatureParameters(signature: string): {r: string; s: string; v: number};
 export function stripHexPrefix(str: string): string;
-export function toNumber(value: number | string | BN): number;
+export function toNumber(value: number | string | BN, bigIntOnOverflow?: boolean): number | string;
 
 // interfaces
 export interface Utils {
@@ -144,7 +144,7 @@ export interface Utils {
     fromUtf8(string: string): string;
     fromWei(value: string | BN, unit?: Unit): string;
     hexToBytes(hex: Hex): number[];
-    hexToNumber(hex: Hex): number;
+    hexToNumber(hex: Hex, bigIntOnOverflow?: boolean): number | string;
     hexToNumberString(hex: Hex): string;
     hexToString(hex: Hex): string;
     hexToUtf8(string: string): string;
@@ -179,7 +179,7 @@ export interface Utils {
     testTopic(bloom: string, topic: string): boolean;
     getSignatureParameters(signature: string): {r: string; s: string; v: number};
     stripHexPrefix(str: string): string;
-    toNumber(value: number | string | BN): number;
+    toNumber(value: number | string | BN, bigIntOnOverflow?: boolean): number | string;
 }
 
 export interface Units {

diff --git a/packages/web3-utils/types/tests/hex-to-number-test.ts b/packages/web3-utils/types/tests/hex-to-number-test.ts
@@ -23,10 +23,14 @@
 import BN = require('bn.js');
 import {hexToNumber} from 'web3-utils';
 
-// $ExpectType number
+// $ExpectType string | number
 hexToNumber('232');
-// $ExpectType number
+// $ExpectType string | number
 hexToNumber(232);
+// $ExpectType string | number
+hexToNumber('0x1fffffffffffff'); // the max safe number 2 ^ 53
+// $ExpectType string | number
+hexToNumber('0x20000000000000', true); // larger than the 2 ^ 53 (unsafe)
 
 // $ExpectError
 hexToNumber(new BN(3));

diff --git a/packages/web3-utils/types/tests/to-number-test.ts b/packages/web3-utils/types/tests/to-number-test.ts
@@ -23,12 +23,16 @@
 import BN = require('bn.js');
 import {toNumber} from 'web3-utils';
 
-// $ExpectType number
+// $ExpectType string | number
 toNumber('234');
-// $ExpectType number
+// $ExpectType string | number
 toNumber(234);
-// $ExpectType number
+// $ExpectType string | number
 toNumber(new BN(3));
+// $ExpectType string | number
+toNumber('0x1fffffffffffff'); // the max safe number 2 ^ 53
+// $ExpectType string | number
+toNumber('0x20000000000000', true); // larger than the 2 ^ 53 (unsafe)
 
 // $ExpectError
 toNumber(['string']);

diff --git a/test/utils.toNumber.js b/test/utils.toNumber.js
@@ -16,25 +16,36 @@ var tests = [
     { value: '-1', expected: -1},
     { value: '-0x1', expected: -1},
     { value: '-15', expected: -15},
-    { value: '-0xf', expected: -15},
-    { value: '0x657468657265756d', expected: '0x657468657265756d', error: true, errorMessage: 'Number can only safely store up to 53 bits'},
-    { value: '0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd', expected: '0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd', error: true, errorMessage: 'Number can only safely store up to 53 bits'},
-    { value: '-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', expected: '-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', error: true, errorMessage: 'Number can only safely store up to 53 bits'},
-    { value: '-0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd', expected: '-0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd', error: true, errorMessage: 'Number can only safely store up to 53 bits'},
+    { value: '-0xf', expected: -15 },
+    { value: '0x657468657265756d', expected: '0x657468657265756d', error: true, errorMessage: 'Number can only safely store up to 53 bits' },
+    { value: '0x657468657265756d', bigIntOnOverflow: true, expected: 7310582880049395053n },
+    { value: '0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd', expected: '0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd', error: true, errorMessage: 'Number can only safely store up to 53 bits' },
+    { value: '0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd', bigIntOnOverflow: true, expected: 115792089237316195423570985008687907853269984665640564039457584007913129639933n },
+    { value: '-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', expected: '-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', error: true, errorMessage: 'Number can only safely store up to 53 bits' },
+    { value: '-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', bigIntOnOverflow: true, expected: -115792089237316195423570985008687907853269984665640564039457584007913129639935n },
+    { value: '-0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd', expected: '-0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd', error: true, errorMessage: 'Number can only safely store up to 53 bits' },
+    { value: '-0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd', bigIntOnOverflow: true, expected: -115792089237316195423570985008687907853269984665640564039457584007913129639933n },
     { value: 0, expected: 0},
     { value: '0', expected: 0},
     { value: '0x0', expected: 0},
     { value: -0, expected: -0},
     { value: '-0', expected: -0},
     { value: '-0x0', expected: -0},
-    { value: [1,2,3,{test: 'data'}], expected: '0x5b312c322c332c7b2274657374223a2264617461227d5d', error: true, errorMessage: 'Number can only safely store up to 53 bits'},
-    { value: {test: 'test'}, expected: '0x7b2274657374223a2274657374227d', error: true, errorMessage: 'Number can only safely store up to 53 bits'},
-    { value: '{"test": "test"}', expected: '0x7b2274657374223a202274657374227d', error: true, errorMessage: 'Number can only safely store up to 53 bits'},
-    { value: 'myString', expected: '0x6d79537472696e67', error: true, errorMessage: 'Number can only safely store up to 53 bits'},
-    { value: 'myString 34534!', expected: '0x6d79537472696e6720333435333421', error: true, errorMessage: 'Number can only safely store up to 53 bits'},
+    { value: [1, 2, 3, { test: 'data' }], expected: '0x5b312c322c332c7b2274657374223a2264617461227d5d', error: true, errorMessage: 'Number can only safely store up to 53 bits' },
+    { value: [1, 2, 3, { test: 'data' }], bigIntOnOverflow: true, expected: 8734466057720693480455376997372198952121265679558147421n },
+    { value: { test: 'test' }, expected: '0x7b2274657374223a2274657374227d', error: true, errorMessage: 'Number can only safely store up to 53 bits' },
+    { value: { test: 'test' }, bigIntOnOverflow: true, expected: 639351337390720496868710369885168253n },
+    { value: '{"test": "test"}', expected: '0x7b2274657374223a202274657374227d', error: true, errorMessage: 'Number can only safely store up to 53 bits' },
+    { value: '{"test": "test"}', bigIntOnOverflow: true, expected: 163673942372024447198222674986970391165n },
+    { value: 'myString', expected: '0x6d79537472696e67', error: true, errorMessage: 'Number can only safely store up to 53 bits' },
+    { value: 'myString', bigIntOnOverflow: true, expected: 7888427981916958311n },
+    { value: 'myString 34534!', expected: '0x6d79537472696e6720333435333421', error: true, errorMessage: 'Number can only safely store up to 53 bits' },
+    { value: 'myString 34534!', bigIntOnOverflow: true, expected: 568421141118403315336782784712881185n },
     { value: new BN(15), expected: 15},
     { value: new BigNumber(15), expected: 15},
-    { value: 'Heeäööä👅D34ɝɣ24Єͽ-.,äü+#/', expected: '0x486565c3a4c3b6c3b6c3a4f09f9185443334c99dc9a33234d084cdbd2d2e2cc3a4c3bc2b232f', error: true, errorMessage: 'Number can only safely store up to 53 bits'},
+    { value: 'Heeäööä👅D34ɝɣ24Єͽ-.,äü+#/', expected: '0x486565c3a4c3b6c3b6c3a4f09f9185443334c99dc9a33234d084cdbd2d2e2cc3a4c3bc2b232f', error: true, errorMessage: 'Number can only safely store up to 53 bits' },
+    { value: 'Heeäööä👅D34ɝɣ24Єͽ-.,äü+#/', bigIntOnOverflow: true, expected: 9217089234592088086444699797948423596835884090143084093263839537376624562728728246738428719n },
+    { value: 'Good', expected: 1198485348},
     { value: true, expected: 1},
     { value: false, expected: 0},
 ];
@@ -45,15 +56,15 @@ describe('lib/utils/utils', function () {
             if (test.error) {
                 it('should error with message', function () {
                     try {
-                        utils.toNumber(test.value)
+                        utils.toNumber(test.value, test.bigIntOnOverflow)
                         assert.fail();
                     } catch(err){
                         assert.strictEqual(err.message, test.errorMessage);
                     }
                 });
             } else {
                 it('should turn ' + test.value + ' to ' + test.expected, function () {
-                    assert.strictEqual(utils.toNumber(test.value), test.expected);
+                    assert.strictEqual(utils.toNumber(test.value, test.bigIntOnOverflow), test.expected);
                 });
             }
         });