From 3d8b8c6a424a1b3ed1989614fa5529591c0bc77f Mon Sep 17 00:00:00 2001 From: station384 Date: Sun, 21 Jan 2018 12:38:01 -0500 Subject: [PATCH 01/13] Add trend_bollinger --- scripts/genetic_backtester/darwin.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/genetic_backtester/darwin.js b/scripts/genetic_backtester/darwin.js index d2ccf8a58f..a594b0f1a7 100755 --- a/scripts/genetic_backtester/darwin.js +++ b/scripts/genetic_backtester/darwin.js @@ -252,6 +252,22 @@ let strategies = { bollinger_upper_bound_pct: RangeFloat(-1, 30), bollinger_lower_bound_pct: RangeFloat(-1, 30) }, + trend_bollinger: { + period_length: RangePeriod(1, 60, 'm'), + markdown_buy_pct: RangeFloat(-1, 5), + markup_sell_pct: RangeFloat(-1, 5), + order_type: RangeMakerTaker(), + sell_stop_pct: Range0(1, 50), + buy_stop_pct: Range0(1, 50), + profit_stop_enable_pct: Range0(1, 20), + profit_stop_pct: Range(1,20), + + // -- strategy + bollinger_size: Range(1, 40), + bollinger_time: RangeFloat(1,6), + bollinger_upper_bound_pct: RangeFloat(-1, 30), + bollinger_lower_bound_pct: RangeFloat(-1, 30) + }, crossover_vwap: { // -- common period_length: RangePeriod(1, 400, 'm'), From 5110ab60b5c292456be2f90538b3441be1773b02 Mon Sep 17 00:00:00 2001 From: station384 Date: Sun, 21 Jan 2018 12:50:41 -0500 Subject: [PATCH 02/13] Add wavetrend strategy --- scripts/genetic_backtester/darwin.js | 26 +++++++++++++++++++++++++ scripts/genetic_backtester/phenotype.js | 2 ++ 2 files changed, 28 insertions(+) diff --git a/scripts/genetic_backtester/darwin.js b/scripts/genetic_backtester/darwin.js index a594b0f1a7..507d93a1ef 100755 --- a/scripts/genetic_backtester/darwin.js +++ b/scripts/genetic_backtester/darwin.js @@ -234,6 +234,12 @@ let RangeNeuralActivation = () => { } return r } +let RangeBoolean = () => { + var r = { + type: 'truefalse' + } + return r +} let strategies = { bollinger: { @@ -542,6 +548,26 @@ let strategies = { down_trend_threshold: Range(0, 50), overbought_rsi_periods: Range(1, 50), overbought_rsi: Range(20, 100) + }, + wavetrend: { + // -- common + period_length: RangePeriod(1, 120, 'm'), + min_periods: Range(1, 200), + markup_pct: RangeFloat(0, 5), + order_type: RangeMakerTaker(), + sell_stop_pct: Range0(1, 50), + buy_stop_pct: Range0(1, 50), + profit_stop_enable_pct: Range0(1, 20), + profit_stop_pct: Range(1,20), + + // -- strategy + wavetrend_channel_length: Range(1,20), + wavetrend_average_length: Range(1,42), + wavetrend_overbought_1: Range(1, 100), + wavetrend_overbought_2: Range(1,100), + wavetrend_oversold_1: Range(-100,0), + wavetrend_oversold_2: Range(-100,0), + wavetrend_trends: RangeBoolean() } } diff --git a/scripts/genetic_backtester/phenotype.js b/scripts/genetic_backtester/phenotype.js index d54c9d46c5..2a7d022b54 100644 --- a/scripts/genetic_backtester/phenotype.js +++ b/scripts/genetic_backtester/phenotype.js @@ -34,6 +34,8 @@ module.exports = { } else if (v.type === 'period_length') { var s = Math.floor((Math.random() * (v.max - v.min + 1)) + v.min) r[k] = s + v.period_length + } else if (v.type === 'truefalse') { + r[k] = (Math.random() > 0.5) ? true : false } } return r From 2eec5590f504ce04b02e53366c66129ef75d5c78 Mon Sep 17 00:00:00 2001 From: station384 Date: Sun, 21 Jan 2018 13:09:25 -0500 Subject: [PATCH 03/13] Add StdDev --- scripts/genetic_backtester/darwin.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/genetic_backtester/darwin.js b/scripts/genetic_backtester/darwin.js index 507d93a1ef..32cf509e62 100755 --- a/scripts/genetic_backtester/darwin.js +++ b/scripts/genetic_backtester/darwin.js @@ -568,6 +568,22 @@ let strategies = { wavetrend_oversold_1: Range(-100,0), wavetrend_oversold_2: Range(-100,0), wavetrend_trends: RangeBoolean() + }, + stddev: { + // -- common + // reference in extensions is given in ms have not heard of an exchange that supports 500ms thru api so setting min at 1 second + period_length: RangePeriod(1, 7200, 's'), + min_periods: Range(1, 2500), + markup_pct: RangeFloat(0, 5), + order_type: RangeMakerTaker(), + sell_stop_pct: Range0(1, 50), + buy_stop_pct: Range0(1, 50), + profit_stop_enable_pct: Range0(1, 20), + profit_stop_pct: Range(1,20), + + // -- strategy + trendtrades_1: Range(2, 20), + trendtrades_2: Range(4, 100) } } From 804d85589a6426ea30416c3cdfaccf6b4dfdd056 Mon Sep 17 00:00:00 2001 From: station384 Date: Sun, 21 Jan 2018 13:14:04 -0500 Subject: [PATCH 04/13] Add Momentum Strategy --- scripts/genetic_backtester/darwin.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/genetic_backtester/darwin.js b/scripts/genetic_backtester/darwin.js index 32cf509e62..2abaecdf01 100755 --- a/scripts/genetic_backtester/darwin.js +++ b/scripts/genetic_backtester/darwin.js @@ -584,6 +584,19 @@ let strategies = { // -- strategy trendtrades_1: Range(2, 20), trendtrades_2: Range(4, 100) + }, + momentum: { + period_length: RangePeriod(1, 120, 'm'), + min_periods: Range(1, 2500), + markup_pct: RangeFloat(0, 5), + order_type: RangeMakerTaker(), + sell_stop_pct: Range0(1, 50), + buy_stop_pct: Range0(1, 50), + profit_stop_enable_pct: Range0(1, 20), + profit_stop_pct: Range(1,20), + + // -- strategy + momentum_size: Range(1,20) } } From 89a568599087a9fded3443b73705144cdfb07185 Mon Sep 17 00:00:00 2001 From: station384 Date: Sun, 21 Jan 2018 22:28:37 -0500 Subject: [PATCH 05/13] Save options to file if backtester_generation has value This is to improve reliability of passing information to backtested scripts. --- commands/sim.js | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/commands/sim.js b/commands/sim.js index 2f96ed0fab..6cc0c90909 100644 --- a/commands/sim.js +++ b/commands/sim.js @@ -38,6 +38,7 @@ module.exports = function container (get, set, clear) { .option('--rsi_periods ', 'number of periods to calculate RSI at', Number, c.rsi_periods) .option('--disable_options', 'disable printing of options') .option('--enable_stats', 'enable printing order stats') + .option('--backtester_generation ','creates a json file in simulations with the generation number', Number, -1) .option('--verbose', 'print status lines on every period') .action(function (selector, cmd) { var s = {options: minimist(process.argv)} @@ -72,6 +73,7 @@ module.exports = function container (get, set, clear) { so.verbose = !!cmd.verbose so.selector = get('lib.objectify-selector')(selector || c.selector) so.mode = 'sim' + if (cmd.conf) { var overrides = require(path.resolve(process.cwd(), cmd.conf)) Object.keys(overrides).forEach(function (k) { @@ -99,10 +101,10 @@ module.exports = function container (get, set, clear) { option_keys.forEach(function (k) { options[k] = so[k] }) - if (so.show_options) { - var options_json = JSON.stringify(options, null, 2) - output_lines.push(options_json) - } + + let options_output = options + options_output.simresults = {} + if (s.my_trades.length) { s.my_trades.push({ price: s.period.close, @@ -112,6 +114,7 @@ module.exports = function container (get, set, clear) { }) } s.balance.currency = n(s.balance.currency).add(n(s.period.close).multiply(s.balance.asset)).format('0.00000000') + s.balance.asset = 0 s.lookback.unshift(s.period) var profit = s.start_capital ? n(s.balance.currency).subtract(s.start_capital).divide(s.start_capital) : n(0) @@ -142,9 +145,31 @@ module.exports = function container (get, set, clear) { output_lines.push('win/loss: ' + (sells - losses) + '/' + losses) output_lines.push('error rate: ' + (sells ? n(losses).divide(sells).format('0.00%') : '0.00%').yellow) } + options_output.simresults.start_capital = s.start_capital; + options_output.simresults.currency = n(s.balance.currency).value(); + options_output.simresults.profit = profit.value(); + options_output.simresults.buy_hold = buy_hold.value(); + options_output.simresults.buy_hold_profit = buy_hold_profit.value(); + options_output.simresults.total_trades = s.my_trades.length; + options_output.simresults.length_days = s.day_count; + options_output.simresults.total_sells = sells; + options_output.simresults.total_losses = losses; + + let options_json = JSON.stringify(options_output, null, 2) + if (so.show_options) { + output_lines.push(options_json) + } + output_lines.forEach(function (line) { console.log(line) }) + + if (so.backtester_generation >= 0) + { + fs.writeFileSync(path.resolve(__dirname, '..', 'simulations', so.selector.normalized+'_'+so.backtester_generation+'.json'),options_json, {encoding: 'utf8'}) + } + + if (so.filename !== 'none') { var html_output = output_lines.map(function (line) { return colors.stripColors(line) @@ -169,6 +194,7 @@ module.exports = function container (get, set, clear) { fs.writeFileSync(out_target, out) console.log('wrote', out_target) } + process.exit(0) } From f4ace06ad2058464f328d078f381d268a17ad56e Mon Sep 17 00:00:00 2001 From: aquamanjl <35556329+aquamanjl@users.noreply.github.com> Date: Mon, 22 Jan 2018 06:01:40 -0500 Subject: [PATCH 06/13] Fixed StochRSI calculations (#1196) * Fix SRSI calculation * Added test case for SRSI. Switched RSI and SRSI output to float and adjusted display accordingly * Oops, disabled all other tests by mistake * Fix test assertion --- .gitignore | 2 + extensions/strategies/srsi_macd/strategy.js | 3 +- lib/rsi.js | 4 +- lib/srsi.js | 85 +++++++++++++++------ test/lib/rsi.test.js | 2 +- test/lib/srsi.test.js | 37 +++++++++ 6 files changed, 107 insertions(+), 26 deletions(-) create mode 100644 test/lib/srsi.test.js diff --git a/.gitignore b/.gitignore index 3bdb77862f..5bad1c04e7 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ temp.html logs .sync dist/* +.idea +*.iml diff --git a/extensions/strategies/srsi_macd/strategy.js b/extensions/strategies/srsi_macd/strategy.js index c184fcba42..11a646e8fc 100644 --- a/extensions/strategies/srsi_macd/strategy.js +++ b/extensions/strategies/srsi_macd/strategy.js @@ -67,7 +67,8 @@ module.exports = function container (get, set, clear) { color = 'red' } cols.push(z(8, n(s.period.macd_histogram).format('+00.0000'), ' ')[color]) - cols.push(z(8, n(s.period.srsi_K).format('000'), ' ').cyan) + cols.push(z(8, n(s.period.srsi_K).format('00.00'), ' ').cyan) + cols.push(z(8, n(s.period.srsi_D).format('00.00'), ' ').yellow) } else { cols.push(' ') diff --git a/lib/rsi.js b/lib/rsi.js index 5e2ca519ef..87ad80ec3f 100644 --- a/lib/rsi.js +++ b/lib/rsi.js @@ -1,3 +1,5 @@ +var mathjs = require('mathjs') + module.exports = function container (get, set, clear) { return function rsi (s, key, length) { if (s.lookback.length >= length) { @@ -32,7 +34,7 @@ module.exports = function container (get, set, clear) { s.period[key] = 100 } else { var rs = s.period[key + '_avg_gain'] / s.period[key + '_avg_loss'] - s.period[key] = Math.round(100 - (100 / (1 + rs))) + s.period[key] = mathjs.round(100 - (100 / (1 + rs)), 2) } } } diff --git a/lib/srsi.js b/lib/srsi.js index ba230c76c4..6fcd05d902 100644 --- a/lib/srsi.js +++ b/lib/srsi.js @@ -1,25 +1,64 @@ -module.exports = function container (get, set, clear) { - return function srsi (s, key, rsi_length, k, d) { - get('lib.rsi')(s, 'rsi', rsi_length) - let RSI = [] - let sum = 0 - if (typeof s.period.rsi !== 'undefined') - s.lookback.slice(0, k).forEach(function (period) { - if (period.rsi) - RSI.push(period.rsi) - }) - - let highestRSI = Math.max(...RSI) - let lowestRSI = Math.min(...RSI) - let stochK = ((s.period.rsi - lowestRSI) / (highestRSI - lowestRSI)) * 100 - - s.lookback.slice(0, d).forEach(period => { - if (period.srsi_K) - sum += period.srsi_K +var mathjs = require('mathjs'); + +module.exports = function container(get, set, clear) { + return function srsi(s, key, rsi_periods, k_periods, d_periods) { + let samplesRequiredForStochRSI = rsi_periods + k_periods + 1; + + if (s.lookback.length >= samplesRequiredForStochRSI - 1) { + let RSI = []; + + if (typeof s.period.rsi !== 'undefined') { + RSI.push(s.period.rsi); + } else { + get('lib.rsi')(s, 'rsi', rsi_periods); + RSI.push(s.period.rsi); + } + + s.lookback.slice(0, samplesRequiredForStochRSI - 1).forEach(function (period) { + if (period.rsi) { + RSI.push(period.rsi); + } }) - let stochD = sum / d - s.period[key + '_K'] = stochK - s.period[key + '_D'] = stochD - //console.log(s.lookback[0]) + + RSI.reverse(); + + if(RSI.length >= samplesRequiredForStochRSI) { + let stochRSI = []; + for(let i = 0; i < (k_periods + d_periods - 1); i++) { + let rsiForPeriod = RSI.slice(i, rsi_periods + i); + let highestRSI = Math.max(...rsiForPeriod); + let lowestRSI = Math.min(...rsiForPeriod); + if(highestRSI == lowestRSI) { + stochRSI.push(0); + } else { + stochRSI.push(((RSI[(rsi_periods - 1) + i] - lowestRSI) / (highestRSI - lowestRSI)) * 100); + } + } + + stochRSI.reverse(); + + let percentK = []; + for(let i = 0; i < k_periods; i++) { + let kData = stochRSI.slice(i, k_periods + i); + if(kData.length == k_periods) { + percentK.push(mathjs.mean(kData)); + } + } + + let percentD = []; + for(let i = 0; i < d_periods; i++) { + let dData = percentK.slice(i, d_periods + i); + if(dData.length == d_periods) { + percentD.push(mathjs.mean(dData)); + } + } + + s.period[key + '_K'] = percentK[0] == 0 ? 0 : mathjs.round(percentK[0], 2); + s.period[key + '_D'] = percentD[0] == 0 ? 0 : mathjs.round(percentD[0], 2); + + //console.log('lib.srsi: For RSI', RSI[RSI.length - 1], '-', '%K is', s.period[key + '_K'], ', %D is', s.period[key + '_D'], ', period info', s.period); + } + } } -} \ No newline at end of file +} + diff --git a/test/lib/rsi.test.js b/test/lib/rsi.test.js index e7f79d9351..8dd63bac2a 100644 --- a/test/lib/rsi.test.js +++ b/test/lib/rsi.test.js @@ -5,7 +5,7 @@ describe('RSI (Relative Strength Index)', function () { it('should calculate RSI with default period', function () { (RSI())(normalData, 'rsi', 14) - expect(normalData.period.rsi).toEqual(32) + expect(normalData.period.rsi).toEqual(32.26) }) it('should set RSI to 100 when there is no losses for the entire period', function() { diff --git a/test/lib/srsi.test.js b/test/lib/srsi.test.js new file mode 100644 index 0000000000..34720133d4 --- /dev/null +++ b/test/lib/srsi.test.js @@ -0,0 +1,37 @@ +var RSI = require('../../lib/rsi'); +var SRSI = require('../../lib/srsi')(RSI); + +describe('SRSI (StochRSI Oscillator)', function () { + + it('should calculate SRSI with default period', function () { + SRSI(data, 'srsi', 14, 3, 3); + + expect(data.period.srsi_K).toEqual(19.38); + expect(data.period.srsi_D).toEqual(23.18); + }) +}); + +var data = { + lookback: [ + {rsi: 64.38}, + {rsi: 66.71}, + {rsi: 70.29}, + {rsi: 66.49}, + {rsi: 71.47}, + {rsi: 76.17}, + {rsi: 83.66}, + {rsi: 81.85}, + {rsi: 82.55}, + {rsi: 82.89}, + {rsi: 78.60}, + {rsi: 64.78}, + {rsi: 64.77}, + {rsi: 70.05}, + {rsi: 68.76}, + {rsi: 69.53}, + {rsi: 70.15} + ].reverse(), + period: { + rsi: 65.61 + } +}; From 688a4b2f9a7d3b7723c89f5a12de8a4068858bfa Mon Sep 17 00:00:00 2001 From: firepol <1702718+firepol@users.noreply.github.com> Date: Mon, 22 Jan 2018 12:03:25 +0100 Subject: [PATCH 07/13] Fix cexio update-products.sh (#1194) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes the error `/usr/bin/env: ‘node\r’: No such file or directory` --- extensions/exchanges/cexio/update-products.sh | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/extensions/exchanges/cexio/update-products.sh b/extensions/exchanges/cexio/update-products.sh index a737821960..b91b75fbd9 100755 --- a/extensions/exchanges/cexio/update-products.sh +++ b/extensions/exchanges/cexio/update-products.sh @@ -1,21 +1,22 @@ -#!/usr/bin/env node -const ccxt = require('ccxt') -const path = require('path') -var client = new ccxt.cex() -client.fetchMarkets().then(result => { - var products = [] - result.forEach(function (product) { - products.push({ - asset: product.info.symbol1, - currency: product.info.symbol2, - min_size: product.info.minLotSize.toString(), - max_size: product.info.maxLotSize != null ? product.info.maxLotSize.toString() : product.info.maxLotSize, - increment: product.info.symbol1 === 'BTC' ? '0.01' : (product.info.symbol2 === 'BTC' ? '0.00000001' : '0.0001'), - label: product.id - }) - }) - var target = path.resolve(__dirname, 'products.json') - require('fs').writeFileSync(target, JSON.stringify(products, null, 2)) - console.log('wrote', target) - process.exit() -}) +#!/usr/bin/env node + +const ccxt = require('ccxt') +const path = require('path') +var client = new ccxt.cex() +client.fetchMarkets().then(result => { + var products = [] + result.forEach(function (product) { + products.push({ + asset: product.info.symbol1, + currency: product.info.symbol2, + min_size: product.info.minLotSize.toString(), + max_size: product.info.maxLotSize != null ? product.info.maxLotSize.toString() : product.info.maxLotSize, + increment: product.info.symbol1 === 'BTC' ? '0.01' : (product.info.symbol2 === 'BTC' ? '0.00000001' : '0.0001'), + label: product.id + }) + }) + var target = path.resolve(__dirname, 'products.json') + require('fs').writeFileSync(target, JSON.stringify(products, null, 2)) + console.log('wrote', target) + process.exit() +}) From 0d361f6dd33d2a8f9fc219816c5709af754f34dd Mon Sep 17 00:00:00 2001 From: station384 Date: Mon, 22 Jan 2018 21:54:13 -0500 Subject: [PATCH 08/13] Sim result passed as file instead of being scraped --- commands/sim.js | 2 +- scripts/genetic_backtester/darwin.js | 121 +++++++++++++++++++++------ 2 files changed, 97 insertions(+), 26 deletions(-) diff --git a/commands/sim.js b/commands/sim.js index 6cc0c90909..c96f16f941 100644 --- a/commands/sim.js +++ b/commands/sim.js @@ -166,7 +166,7 @@ module.exports = function container (get, set, clear) { if (so.backtester_generation >= 0) { - fs.writeFileSync(path.resolve(__dirname, '..', 'simulations', so.selector.normalized+'_'+so.backtester_generation+'.json'),options_json, {encoding: 'utf8'}) + fs.writeFileSync(path.resolve(__dirname, '..', 'simulations', so.selector.normalized.toLowerCase()+'_'+so.backtester_generation+'.json'),options_json, {encoding: 'utf8'}) } diff --git a/scripts/genetic_backtester/darwin.js b/scripts/genetic_backtester/darwin.js index 2abaecdf01..0f1eefac3f 100755 --- a/scripts/genetic_backtester/darwin.js +++ b/scripts/genetic_backtester/darwin.js @@ -15,10 +15,11 @@ let fs = require('fs') let GeneticAlgorithmCtor = require('geneticalgorithm') let StripAnsi = require('strip-ansi') let moment = require('moment') +let path = require('path') let Phenotypes = require('./phenotype.js') -let VERSION = 'Zenbot 4 Genetic Backtester v0.2' +let VERSION = 'Zenbot 4 Genetic Backtester v0.2.1' let PARALLEL_LIMIT = (process.env.PARALLEL_LIMIT && +process.env.PARALLEL_LIMIT) || require('os').cpus().length @@ -40,6 +41,7 @@ let iterationCount = 0 let runCommand = (taskStrategyName, phenotype, cb) => { var cmdArgs = Object.assign({}, phenotype) + //cmdArgs.backtester_generation = cb.scope cmdArgs.strategy = taskStrategyName Object.assign(cmdArgs, simArgs) @@ -70,7 +72,7 @@ let runCommand = (taskStrategyName, phenotype, cb) => { let result = null try { - result = processOutput(stdout) + result = processOutput(stdout,phenotype) phenotype['sim'] = result result['fitness'] = Phenotypes.fitness(phenotype) } catch (err) { @@ -94,7 +96,7 @@ let runUpdate = (days, selector) => { }) } -let processOutput = output => { +let processOutput = (output, pheno)=> { let jsonRegexp = /(\{[\s\S]*?\})\send balance/g let endBalRegexp = /end balance: (\d+\.\d+) \(/g let buyHoldRegexp = /buy hold: (\d+\.\d+) \(/g @@ -104,27 +106,70 @@ let processOutput = output => { let strippedOutput = StripAnsi(output) let output2 = strippedOutput.substr(strippedOutput.length - 3500) + + let tFileName = path.resolve(__dirname, '..','..', 'simulations', pheno.exchangeMarketPair.toLowerCase()+'_'+pheno.backtester_generation+'.json') + let simulationResults + + let params + let endBalance + let buyHold + let vsBuyHold + let wlMatch + let errMatch + let wins + let losses + let errorRate + let days + let start + let end + // This can retrieve the results 2 different places. It defaults to reading it from the json file + // but if no file is found it will fall back to the older metheod of scraping the output of the sim process + if (fs.existsSync(tFileName)) + { + let jsonBuffer + jsonBuffer = fs.readFileSync(tFileName,{encoding:'utf8'}) + simulationResults = JSON.parse(jsonBuffer) + fs.unlinkSync(tFileName) + } + + + if (typeof(simulationResults) === 'object' ) + { + params = simulationResults + endBalance = simulationResults.simresults.currency + buyHold = simulationResults.simresults.buy_hold + vsBuyHold = simulationResults.simresults.buy_hold + wlMatch = (simulationResults.simresults.total_sells - simulationResults.simresults.total_losses) +'/'+ simulationResults.simresults.total_losses + wins = simulationResults.simresults.total_sells + losses = simulationResults.simresults.total_losses + errorRate = simulationResults.simresults.total_losses / simulationResults.simresults.total_sells + days = parseInt(simulationResults.days) + start = parseInt(simulationResults.start) + end = parseInt(simulationResults.end || null) + } + else + { + let rawParams = jsonRegexp.exec(output2)[1] + params = JSON.parse(rawParams) + endBalance = endBalRegexp.exec(output2)[1] + buyHold = buyHoldRegexp.exec(output2)[1] + vsBuyHold = vsBuyHoldRegexp.exec(output2)[1] + wlMatch = wlRegexp.exec(output2) + errMatch = errRegexp.exec(output2) + wins = wlMatch !== null ? parseInt(wlMatch[1]) : 0 + losses = wlMatch !== null ? parseInt(wlMatch[2]) : 0 + errorRate = errMatch !== null ? parseInt(errMatch[1]) : 0 + days = parseInt(params.days) + start = parseInt(params.start) + end = parseInt(params.end) + } + + let roi = roundp(((endBalance - params.currency_capital) / params.currency_capital) * 100, 3 ) + - let rawParams = jsonRegexp.exec(output2)[1] - let params = JSON.parse(rawParams) - let endBalance = endBalRegexp.exec(output2)[1] - let buyHold = buyHoldRegexp.exec(output2)[1] - let vsBuyHold = vsBuyHoldRegexp.exec(output2)[1] - let wlMatch = wlRegexp.exec(output2) - let errMatch = errRegexp.exec(output2) - let wins = wlMatch !== null ? parseInt(wlMatch[1]) : 0 - let losses = wlMatch !== null ? parseInt(wlMatch[2]) : 0 - let errorRate = errMatch !== null ? parseInt(errMatch[1]) : 0 - let days = parseInt(params.days) - let start = parseInt(params.start) - let end = parseInt(params.end) - - let roi = roundp( - ((endBalance - params.currency_capital) / params.currency_capital) * 100, - 3 - ) - - let r = JSON.parse(rawParams.replace(/[\r\n]/g, '')) + //todo: figure out what this is trying to do. + let r = params + //JSON.parse(rawParams.replace(/[\r\n]/g, '')) delete r.asset_capital delete r.buy_pct delete r.currency_capital @@ -139,6 +184,7 @@ let processOutput = output => { delete r.stats delete r.use_strategies delete r.verbose + delete r.simresults r.selector = r.selector.normalized if (start) { @@ -243,7 +289,7 @@ let RangeBoolean = () => { let strategies = { bollinger: { - period_length: RangePeriod(1, 60, 'm'), + period_length: RangePeriod(1, 120, 'm'), markdown_buy_pct: RangeFloat(-1, 5), markup_sell_pct: RangeFloat(-1, 5), order_type: RangeMakerTaker(), @@ -259,7 +305,7 @@ let strategies = { bollinger_lower_bound_pct: RangeFloat(-1, 30) }, trend_bollinger: { - period_length: RangePeriod(1, 60, 'm'), + period_length: RangePeriod(1, 120, 'm'), markdown_buy_pct: RangeFloat(-1, 5), markup_sell_pct: RangeFloat(-1, 5), order_type: RangeMakerTaker(), @@ -636,6 +682,29 @@ let selectedStrategies = (strategyName === 'all') ? allStrategyNames() : strateg let importedPoolData = (populationFileName) ? JSON.parse(fs.readFileSync(populationFileName, 'utf8')) : null +//Clean up any generation files left over in the simulation directory +//they will be overwritten, but best not to confuse the issue. +//if it fails. doesn't matter they will be overwritten anyways. not need to halt the system. +try +{ + let tDirName = path.resolve(__dirname, '..','..', 'simulations') + let tFileName = simArgs.selector.toLowerCase()+'_' + let files = fs.readdirSync(tDirName) + + for(let i = 0; i < files.length; i++) + { + if (files[i].lastIndexOf(tFileName) == 0) + { + let filePath = path.resolve(__dirname, '..','..', 'simulations',files[i] ) + fs.unlinkSync(filePath) + } + + } +} catch (err) +{ + console.log('error deleting lint from prior run') +} + selectedStrategies.forEach(function(v) { let strategyPool = pools[v] = {} @@ -722,6 +791,8 @@ let simulateGeneration = () => { iterationCount = 1 let tasks = selectedStrategies.map(v => pools[v]['pool'].population().map(phenotype => { return cb => { + phenotype.backtester_generation = iterationCount + phenotype.exchangeMarketPair = argv.selector runCommand(v, phenotype, cb) } })).reduce((a, b) => a.concat(b)) From 28efeba5c644d92ae52e0a2ad25dbfe4652864d2 Mon Sep 17 00:00:00 2001 From: station384 Date: Mon, 22 Jan 2018 22:54:49 -0500 Subject: [PATCH 09/13] Cleanup code and comments --- scripts/genetic_backtester/darwin.js | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/scripts/genetic_backtester/darwin.js b/scripts/genetic_backtester/darwin.js index 0f1eefac3f..5da8bf0a78 100755 --- a/scripts/genetic_backtester/darwin.js +++ b/scripts/genetic_backtester/darwin.js @@ -39,9 +39,18 @@ let NEUTRAL_RATE_AUTO = false let iterationCount = 0 +//todo: remove these and anything that uses them after verification that new system will work in all operating systems +//note compiling regex is costly on cpu and memory (realativly speaking), they should be made const, and top level to avoid +//destruction and recreation. save a little time and memory. +const jsonRegexp = /(\{[\s\S]*?\})\send balance/g +const endBalRegexp = /end balance: (\d+\.\d+) \(/g +const buyHoldRegexp = /buy hold: (\d+\.\d+) \(/g +const vsBuyHoldRegexp = /vs. buy hold: (-?\d+\.\d+)%/g +const wlRegexp = /win\/loss: (\d+)\/(\d+)/g +const errRegexp = /error rate: (.*)%/g + let runCommand = (taskStrategyName, phenotype, cb) => { var cmdArgs = Object.assign({}, phenotype) - //cmdArgs.backtester_generation = cb.scope cmdArgs.strategy = taskStrategyName Object.assign(cmdArgs, simArgs) @@ -97,12 +106,7 @@ let runUpdate = (days, selector) => { } let processOutput = (output, pheno)=> { - let jsonRegexp = /(\{[\s\S]*?\})\send balance/g - let endBalRegexp = /end balance: (\d+\.\d+) \(/g - let buyHoldRegexp = /buy hold: (\d+\.\d+) \(/g - let vsBuyHoldRegexp = /vs. buy hold: (-?\d+\.\d+)%/g - let wlRegexp = /win\/loss: (\d+)\/(\d+)/g - let errRegexp = /error rate: (.*)%/g + let strippedOutput = StripAnsi(output) let output2 = strippedOutput.substr(strippedOutput.length - 3500) @@ -122,8 +126,10 @@ let processOutput = (output, pheno)=> { let days let start let end - // This can retrieve the results 2 different places. It defaults to reading it from the json file + // This can retrieve the results from 2 different places. It defaults to reading it from the json file // but if no file is found it will fall back to the older metheod of scraping the output of the sim process + // stdio scraping to be removed after full verification of functionality. + // todo: see above comment if (fs.existsSync(tFileName)) { let jsonBuffer @@ -169,7 +175,6 @@ let processOutput = (output, pheno)=> { //todo: figure out what this is trying to do. let r = params - //JSON.parse(rawParams.replace(/[\r\n]/g, '')) delete r.asset_capital delete r.buy_pct delete r.currency_capital From a8099ed4c6520a64667dfb93d1aac97672693b8a Mon Sep 17 00:00:00 2001 From: station384 Date: Tue, 23 Jan 2018 09:40:52 -0500 Subject: [PATCH 10/13] Ignore the sim outputs --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 5bad1c04e7..f80f1f6188 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ logs dist/* .idea *.iml +simulations/*_*.json + From 7d8c41a056050025a3a6d9992358128585efa3a4 Mon Sep 17 00:00:00 2001 From: station384 Date: Tue, 23 Jan 2018 11:06:10 -0500 Subject: [PATCH 11/13] Correct mistake in vsBuyHold Missed return result of vsBuyHold in the Sim command JSON results. vsBuyHold needs to be a percentage represented by whole numbers. i.e. 99.5 instead of 0.995. --- commands/sim.js | 19 ++++++++++--------- scripts/genetic_backtester/darwin.js | 7 ++++--- scripts/genetic_backtester/phenotype.js | 1 + 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/commands/sim.js b/commands/sim.js index c96f16f941..6241cd1112 100644 --- a/commands/sim.js +++ b/commands/sim.js @@ -145,15 +145,16 @@ module.exports = function container (get, set, clear) { output_lines.push('win/loss: ' + (sells - losses) + '/' + losses) output_lines.push('error rate: ' + (sells ? n(losses).divide(sells).format('0.00%') : '0.00%').yellow) } - options_output.simresults.start_capital = s.start_capital; - options_output.simresults.currency = n(s.balance.currency).value(); - options_output.simresults.profit = profit.value(); - options_output.simresults.buy_hold = buy_hold.value(); - options_output.simresults.buy_hold_profit = buy_hold_profit.value(); - options_output.simresults.total_trades = s.my_trades.length; - options_output.simresults.length_days = s.day_count; - options_output.simresults.total_sells = sells; - options_output.simresults.total_losses = losses; + options_output.simresults.start_capital = s.start_capital + options_output.simresults.currency = n(s.balance.currency).value() + options_output.simresults.profit = profit.value() + options_output.simresults.buy_hold = buy_hold.value() + options_output.simresults.buy_hold_profit = buy_hold_profit.value() + options_output.simresults.total_trades = s.my_trades.length + options_output.simresults.length_days = s.day_count + options_output.simresults.total_sells = sells + options_output.simresults.total_losses = losses + options_output.simresults.vs_buy_hold = n(s.balance.currency).subtract(buy_hold).divide(buy_hold).value() * 100.00 let options_json = JSON.stringify(options_output, null, 2) if (so.show_options) { diff --git a/scripts/genetic_backtester/darwin.js b/scripts/genetic_backtester/darwin.js index 5da8bf0a78..f3423d1735 100755 --- a/scripts/genetic_backtester/darwin.js +++ b/scripts/genetic_backtester/darwin.js @@ -144,7 +144,7 @@ let processOutput = (output, pheno)=> { params = simulationResults endBalance = simulationResults.simresults.currency buyHold = simulationResults.simresults.buy_hold - vsBuyHold = simulationResults.simresults.buy_hold + vsBuyHold = simulationResults.simresults.vs_buy_hold wlMatch = (simulationResults.simresults.total_sells - simulationResults.simresults.total_losses) +'/'+ simulationResults.simresults.total_losses wins = simulationResults.simresults.total_sells losses = simulationResults.simresults.total_losses @@ -202,11 +202,11 @@ let processOutput = (output, pheno)=> { r.days = params.days } - return { + let results = { params: 'module.exports = ' + JSON.stringify(r), endBalance: parseFloat(endBalance), buyHold: parseFloat(buyHold), - vsBuyHold: parseFloat(vsBuyHold), + vsBuyHold: parseFloat(vsBuyHold) || vsBuyHold, wins: wins, losses: losses, errorRate: parseFloat(errorRate), @@ -222,6 +222,7 @@ let processOutput = (output, pheno)=> { strategy: params.strategy, frequency: roundp((wins + losses) / days, 3) } + return results } let Range = (min, max) => { diff --git a/scripts/genetic_backtester/phenotype.js b/scripts/genetic_backtester/phenotype.js index 2a7d022b54..a0a7baef69 100644 --- a/scripts/genetic_backtester/phenotype.js +++ b/scripts/genetic_backtester/phenotype.js @@ -71,6 +71,7 @@ module.exports = { var vsBuyHoldRate = (phenotype.sim.vsBuyHold / 50) var wlRatio = phenotype.sim.wins - phenotype.sim.losses + // 2.71828 is https://en.wikipedia.org/wiki/E_(mathematical_constant) var wlRatioRate = 1.0 / (1.0 + Math.pow(2.71828, wlRatio < 0 ? wlRatio:-(wlRatio))) var rate = vsBuyHoldRate * (wlRatioRate) return rate From 6b1c8dc8d7ccde1b5e7673e64026a9286e7492e1 Mon Sep 17 00:00:00 2001 From: station384 Date: Tue, 23 Jan 2018 16:49:24 -0500 Subject: [PATCH 12/13] Alter auto_backtester to use json output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit changed json file name to new format sim___.json if selector or marketExcahngePair contains ‘_’ it will be stripped this allows for easier parsing of the file name. Final additions or I’ll just keep tweaking it. --- .gitignore | 2 +- commands/sim.js | 4 +- scripts/auto_backtester/backtester.js | 176 +++++++++++++++----------- scripts/genetic_backtester/darwin.js | 35 +---- 4 files changed, 109 insertions(+), 108 deletions(-) diff --git a/.gitignore b/.gitignore index f80f1f6188..1aca3b8fba 100644 --- a/.gitignore +++ b/.gitignore @@ -32,5 +32,5 @@ logs dist/* .idea *.iml -simulations/*_*.json +simulations/sim_*.json diff --git a/commands/sim.js b/commands/sim.js index 6241cd1112..3b8b1860df 100644 --- a/commands/sim.js +++ b/commands/sim.js @@ -84,7 +84,7 @@ module.exports = function container (get, set, clear) { if (!so.min_periods) so.min_periods = 1 var cursor, reversing, reverse_point var query_start = so.start ? tb(so.start).resize(so.period_length).subtract(so.min_periods + 2).toMilliseconds() : null - + function exitSim () { console.log() if (!s.period) { @@ -167,7 +167,7 @@ module.exports = function container (get, set, clear) { if (so.backtester_generation >= 0) { - fs.writeFileSync(path.resolve(__dirname, '..', 'simulations', so.selector.normalized.toLowerCase()+'_'+so.backtester_generation+'.json'),options_json, {encoding: 'utf8'}) + fs.writeFileSync(path.resolve(__dirname, '..', 'simulations','sim_'+so.strategy.replace('_','')+'_'+ so.selector.normalized.replace('_','').toLowerCase()+'_'+so.backtester_generation+'.json'),options_json, {encoding: 'utf8'}) } diff --git a/scripts/auto_backtester/backtester.js b/scripts/auto_backtester/backtester.js index 8f21a1ab80..315853e77a 100755 --- a/scripts/auto_backtester/backtester.js +++ b/scripts/auto_backtester/backtester.js @@ -16,11 +16,12 @@ let parallel = require('run-parallel-limit'); let json2csv = require('json2csv'); let roundp = require('round-precision'); let fs = require('fs'); +let path = require('path') let StripAnsi = require('strip-ansi'); let VERSION = 'Zenbot 4.04 Backtester v0.2'; -let PARALLEL_LIMIT = require('os').cpus().length; +let PARALLEL_LIMIT = 3//require('os').cpus().length; let TREND_EMA_MIN = 20; let TREND_EMA_MAX = 20; @@ -70,7 +71,7 @@ let objectProduct = obj => { }); }; -let runCommand = (strategy, cb) => { +let runCommand = (strategy, exchangeMarketPair,strategyName, cb) => { countArr.push(1); let strategyArgs = { cci_srsi: `--cci_periods=${strategy.rsi_periods} --rsi_periods=${strategy.srsi_periods} --srsi_periods=${strategy.srsi_periods} --srsi_k=${strategy.srsi_k} --srsi_d=${strategy.srsi_d} --oversold_rsi=${strategy.oversold_rsi} --overbought_rsi=${strategy.overbought_rsi} --oversold_cci=${strategy.oversold_cci} --overbought_cci=${strategy.overbought_cci} --constant=${strategy.constant}`, @@ -82,8 +83,11 @@ let runCommand = (strategy, cb) => { trend_ema: `--trend_ema=${strategy.trend_ema} --oversold_rsi=${strategy.oversold_rsi} --oversold_rsi_periods=${strategy.oversold_rsi_periods} --neutral_rate=${strategy.neutral_rate}` }; let zenbot_cmd = process.platform === 'win32' ? 'zenbot.bat' : './zenbot.sh'; // Use 'win32' for 64 bit windows too - let command = `${zenbot_cmd} sim ${simArgs} ${strategyArgs[strategyName]} --period_length=${strategy.period_length} --min_periods=${strategy.min_periods}`; - console.log(`[ ${countArr.length}/${strategies[strategyName].length} ] ${command}`); + let localGen = countArr.length + strategy.backtester_generation = localGen + + let command = `${zenbot_cmd} sim ${simArgs} ${strategyArgs[strategyName]} --period_length=${strategy.period_length} --min_periods=${strategy.min_periods} --backtester_generation=${localGen}`; + console.log(`[ ${localGen}/${strategies[strategyName].length} ] ${command}`); shell.exec(command, {silent:true, async:true}, (code, stdout, stderr) => { if (code) { @@ -91,40 +95,36 @@ let runCommand = (strategy, cb) => { console.error(stderr) return cb(null, null) } - cb(null, processOutput(stdout)); + cb(null, processOutput(stdout,localGen,strategyName, exchangeMarketPair)); }); }; -let processOutput = output => { - let jsonRegexp = /(\{[\s\S]*?\})\send balance/g; - let endBalRegexp = /end balance: (\d+\.\d+) \(/g; - let buyHoldRegexp = /buy hold: (\d+\.\d+) \(/g; - let vsBuyHoldRegexp = /vs. buy hold: (-?\d+\.\d+)%/g; - let wlRegexp = /win\/loss: (\d+)\/(\d+)/g; - let errRegexp = /error rate: (.*)%/g; - - let strippedOutput = StripAnsi(output); - let output2 = strippedOutput.substr(strippedOutput.length - 3500); - - let rawParams = jsonRegexp.exec(output2)[1]; - let params = JSON.parse(rawParams); - let endBalance = endBalRegexp.exec(output2)[1]; - let buyHold = buyHoldRegexp.exec(output2)[1]; - let vsBuyHold = vsBuyHoldRegexp.exec(output2)[1]; - let wlMatch = wlRegexp.exec(output2); - let errMatch = errRegexp.exec(output2); - let wins = wlMatch !== null ? parseInt(wlMatch[1]) : 0; - let losses = wlMatch !== null ? parseInt(wlMatch[2]) : 0; - let errorRate = errMatch !== null ? parseInt(errMatch[1]) : 0; - let days = parseInt(params.days); +let processOutput = (output,generation,strategyName, exchangeMarketPair) => { + + let tFileName = path.resolve(__dirname, '..','..', 'simulations','sim_'+strategyName.replace('_','')+'_'+exchangeMarketPair.replace(' ','')+'_'+generation+'.json') + let simulationResults + if (fs.existsSync(tFileName)) + { + let jsonBuffer + jsonBuffer = fs.readFileSync(tFileName,{encoding:'utf8'}) + simulationResults = JSON.parse(jsonBuffer) + fs.unlinkSync(tFileName) + } + let endBalance = simulationResults.simresults.currency + let buyHold = simulationResults.simresults.buy_hold + let vsBuyHold = simulationResults.simresults.vs_buy_hold + let wins = simulationResults.simresults.total_sells + let losses = simulationResults.simresults.total_losses + let errorRate = simulationResults.simresults.total_losses / simulationResults.simresults.total_sells + let days = parseInt(simulationResults.days) let roi = roundp( - ((endBalance - params.currency_capital) / params.currency_capital) * 100, + ((endBalance - simulationResults.currency_capital) / simulationResults.currency_capital) * 100, 3 - ); + ) return { - params: rawParams.replace(/[\r\n]/g, ''), + params: JSON.stringify(simulationResults), endBalance: parseFloat(endBalance), buyHold: parseFloat(buyHold), vsBuyHold: parseFloat(vsBuyHold), @@ -133,64 +133,64 @@ let processOutput = output => { errorRate: parseFloat(errorRate), // cci_srsi - cciPeriods: params.cci_periods, - rsiPeriods: params.rsi_periods, - srsiPeriods: params.srsi_periods, - srsiK: params.srsi_k, - srsiD: params.srsi_d, - oversoldRsi: params.oversold_rsi, - overboughtRsi: params.overbought_rsi, - oversoldCci: params.oversold_cci, - overboughtCci: params.overbought_cci, - constant: params.consant, + cciPeriods: simulationResults.cci_periods, + rsiPeriods: simulationResults.rsi_periods, + srsiPeriods: simulationResults.srsi_periods, + srsiK: simulationResults.srsi_k, + srsiD: simulationResults.srsi_d, + oversoldRsi: simulationResults.oversold_rsi, + overboughtRsi: simulationResults.overbought_rsi, + oversoldCci: simulationResults.oversold_cci, + overboughtCci: simulationResults.overbought_cci, + constant: simulationResults.consant, // srsi_macd - rsiPeriods: params.rsi_periods, - srsiPeriods: params.srsi_periods, - srsiK: params.srsi_k, - srsiD: params.srsi_d, - oversoldRsi: params.oversold_rsi, - overboughtRsi: params.overbought_rsi, - emaShortPeriod: params.ema_short_period, - emaLongPeriod: params.ema_long_period, - signalPeriod: params.signal_period, - upTrendThreshold: params.up_trend_threshold, - downTrendThreshold: params.down_trend_threshold, + rsiPeriods: simulationResults.rsi_periods, + srsiPeriods: simulationResults.srsi_periods, + srsiK: simulationResults.srsi_k, + srsiD: simulationResults.srsi_d, + oversoldRsi: simulationResults.oversold_rsi, + overboughtRsi: simulationResults.overbought_rsi, + emaShortPeriod: simulationResults.ema_short_period, + emaLongPeriod: simulationResults.ema_long_period, + signalPeriod: simulationResults.signal_period, + upTrendThreshold: simulationResults.up_trend_threshold, + downTrendThreshold: simulationResults.down_trend_threshold, // macd - emaShortPeriod: params.ema_short_period, - emaLongPeriod: params.ema_long_period, - signalPeriod: params.signal_period, - upTrendThreshold: params.up_trend_threshold, - downTrendThreshold: params.down_trend_threshold, - overboughtRsiPeriods: params.overbought_rsi_periods, - overboughtRsi: params.overbought_rsi, + emaShortPeriod: simulationResults.ema_short_period, + emaLongPeriod: simulationResults.ema_long_period, + signalPeriod: simulationResults.signal_period, + upTrendThreshold: simulationResults.up_trend_threshold, + downTrendThreshold: simulationResults.down_trend_threshold, + overboughtRsiPeriods: simulationResults.overbought_rsi_periods, + overboughtRsi: simulationResults.overbought_rsi, // rsi - rsiPeriods: params.rsi_periods, - oversoldRsi: params.oversold_rsi, - overboughtRsi: params.overbought_rsi, - rsiRecover: params.rsi_recover, - rsiDrop: params.rsi_drop, - rsiDivsor: params.rsi_divisor, + rsiPeriods: simulationResults.rsi_periods, + oversoldRsi: simulationResults.oversold_rsi, + overboughtRsi: simulationResults.overbought_rsi, + rsiRecover: simulationResults.rsi_recover, + rsiDrop: simulationResults.rsi_drop, + rsiDivsor: simulationResults.rsi_divisor, // sar - sarAf: params.sar_af, - sarMaxAf: params.sar_max_af, + sarAf: simulationResults.sar_af, + sarMaxAf: simulationResults.sar_max_af, // speed - baselinePeriods: params.baseline_periods, - triggerFactor: params.trigger_factor, + baselinePeriods: simulationResults.baseline_periods, + triggerFactor: simulationResults.trigger_factor, // trend_ema - trendEma: params.trend_ema, - neutralRate: params.neutral_rate, - oversoldRsiPeriods: params.oversold_rsi_periods, - oversoldRsi: params.oversold_rsi, + trendEma: simulationResults.trend_ema, + neutralRate: simulationResults.neutral_rate, + oversoldRsiPeriods: simulationResults.oversold_rsi_periods, + oversoldRsi: simulationResults.oversold_rsi, days: days, - period_length: params.period_length, - min_periods: params.min_periods, + period_length: simulationResults.period_length, + min_periods: simulationResults.min_periods, roi: roi, wlRatio: losses > 0 ? roundp(wins / losses, 3) : 'Infinity', frequency: roundp((wins + losses) / days, 3) @@ -270,6 +270,7 @@ let strategies = { }; let args = process.argv; +let exchangeMarketPair = args[2].toLowerCase(); args.shift(); args.shift(); let simArgs = args.join(' '); @@ -280,7 +281,7 @@ if (args.indexOf('--strategy') !== -1) { let tasks = strategies[strategyName].map(strategy => { return cb => { - runCommand(strategy, cb) + runCommand(strategy,exchangeMarketPair,strategyName.toLowerCase(), cb) } }); @@ -288,8 +289,31 @@ console.log(`\n--==${VERSION}==--`); console.log(new Date().toUTCString()); console.log(`\nBacktesting [${strategies[strategyName].length}] iterations for strategy ${strategyName}...\n`); +//Clean up any generation files left over in the simulation directory +//they will be overwritten, but best not to confuse the issue. +//if it fails. doesn't matter they will be overwritten anyways. not need to halt the system. +try +{ + let tDirName = path.resolve(__dirname, '..','..', 'simulations') + let tFileName = 'sim_' + let files = fs.readdirSync(tDirName) + + for(let i = 0; i < files.length; i++) + { + if (files[i].lastIndexOf(tFileName) == 0) + { + let filePath = path.resolve(__dirname, '..','..', 'simulations',files[i] ) + fs.unlinkSync(filePath) + } + + } +} catch (err) +{ + console.log('error deleting lint from prior run') +} + parallel(tasks, PARALLEL_LIMIT, (err, results) => { - console.log("\nBacktesting complete, saving results..."); + console.log('\nBacktesting complete, saving results...'); results = results.filter(function (r) { return !!r }) diff --git a/scripts/genetic_backtester/darwin.js b/scripts/genetic_backtester/darwin.js index f3423d1735..cf48a7f22f 100755 --- a/scripts/genetic_backtester/darwin.js +++ b/scripts/genetic_backtester/darwin.js @@ -39,15 +39,6 @@ let NEUTRAL_RATE_AUTO = false let iterationCount = 0 -//todo: remove these and anything that uses them after verification that new system will work in all operating systems -//note compiling regex is costly on cpu and memory (realativly speaking), they should be made const, and top level to avoid -//destruction and recreation. save a little time and memory. -const jsonRegexp = /(\{[\s\S]*?\})\send balance/g -const endBalRegexp = /end balance: (\d+\.\d+) \(/g -const buyHoldRegexp = /buy hold: (\d+\.\d+) \(/g -const vsBuyHoldRegexp = /vs. buy hold: (-?\d+\.\d+)%/g -const wlRegexp = /win\/loss: (\d+)\/(\d+)/g -const errRegexp = /error rate: (.*)%/g let runCommand = (taskStrategyName, phenotype, cb) => { var cmdArgs = Object.assign({}, phenotype) @@ -68,6 +59,7 @@ let runCommand = (taskStrategyName, phenotype, cb) => { console.log(`[ ${iterationCount++}/${populationSize * selectedStrategies.length} ] ${command}`) phenotype['sim'] = {} + shell.exec(command, { silent: true, @@ -81,7 +73,7 @@ let runCommand = (taskStrategyName, phenotype, cb) => { let result = null try { - result = processOutput(stdout,phenotype) + result = processOutput(stdout,taskStrategyName,phenotype) phenotype['sim'] = result result['fitness'] = Phenotypes.fitness(phenotype) } catch (err) { @@ -105,13 +97,13 @@ let runUpdate = (days, selector) => { }) } -let processOutput = (output, pheno)=> { +let processOutput = (output,taskStrategyName, pheno)=> { let strippedOutput = StripAnsi(output) let output2 = strippedOutput.substr(strippedOutput.length - 3500) - let tFileName = path.resolve(__dirname, '..','..', 'simulations', pheno.exchangeMarketPair.toLowerCase()+'_'+pheno.backtester_generation+'.json') + let tFileName = path.resolve(__dirname, '..','..', 'simulations','sim_'+taskStrategyName.replace('_','')+'_'+ pheno.exchangeMarketPair.toLowerCase().replace('_','')+'_'+pheno.backtester_generation+'.json') let simulationResults let params @@ -153,22 +145,7 @@ let processOutput = (output, pheno)=> { start = parseInt(simulationResults.start) end = parseInt(simulationResults.end || null) } - else - { - let rawParams = jsonRegexp.exec(output2)[1] - params = JSON.parse(rawParams) - endBalance = endBalRegexp.exec(output2)[1] - buyHold = buyHoldRegexp.exec(output2)[1] - vsBuyHold = vsBuyHoldRegexp.exec(output2)[1] - wlMatch = wlRegexp.exec(output2) - errMatch = errRegexp.exec(output2) - wins = wlMatch !== null ? parseInt(wlMatch[1]) : 0 - losses = wlMatch !== null ? parseInt(wlMatch[2]) : 0 - errorRate = errMatch !== null ? parseInt(errMatch[1]) : 0 - days = parseInt(params.days) - start = parseInt(params.start) - end = parseInt(params.end) - } + let roi = roundp(((endBalance - params.currency_capital) / params.currency_capital) * 100, 3 ) @@ -694,7 +671,7 @@ let importedPoolData = (populationFileName) ? JSON.parse(fs.readFileSync(populat try { let tDirName = path.resolve(__dirname, '..','..', 'simulations') - let tFileName = simArgs.selector.toLowerCase()+'_' + let tFileName = 'sim_' let files = fs.readdirSync(tDirName) for(let i = 0; i < files.length; i++) From 5cf89065380bef6f8785e8b5428d50694dfac61b Mon Sep 17 00:00:00 2001 From: station384 Date: Wed, 24 Jan 2018 09:32:52 -0500 Subject: [PATCH 13/13] Remove debug code now matches genetic_backtester. --- scripts/auto_backtester/backtester.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_backtester/backtester.js b/scripts/auto_backtester/backtester.js index 315853e77a..628c2806bf 100755 --- a/scripts/auto_backtester/backtester.js +++ b/scripts/auto_backtester/backtester.js @@ -21,7 +21,7 @@ let StripAnsi = require('strip-ansi'); let VERSION = 'Zenbot 4.04 Backtester v0.2'; -let PARALLEL_LIMIT = 3//require('os').cpus().length; +let PARALLEL_LIMIT = (process.env.PARALLEL_LIMIT && +process.env.PARALLEL_LIMIT) || require('os').cpus().length; let TREND_EMA_MIN = 20; let TREND_EMA_MAX = 20;