From 627c40879de43b9e5d222f9351fdb224ecc974b0 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 14:55:08 -0400 Subject: [PATCH 01/13] Calculate length as constant --- README.md | 4 +++- eureqa.jl | 29 ++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9acebc4c..3ca2511a 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ pd.DataFrame, Results dataframe, giving complexity, MSE, and equations # TODO - [ ] Make scaling of changes to constant a hyperparameter -- [ ] Update hall of fame every iteration +- [ ] Update hall of fame every iteration? - [ ] Calculate feature importances of future mutations, by looking at correlation between residual of model, and the features. - Store feature importances of future, and periodically update it. - [ ] Implement more parts of the original Eureqa algorithms: https://www.creativemachineslab.com/eureqa.html @@ -162,6 +162,8 @@ pd.DataFrame, Results dataframe, giving complexity, MSE, and equations - Current most expensive operations: - [ ] Calculating the loss function - there is duplicate calculations happening. - [x] Declaration of the weights array every iteration +- [x] Add a node at the top of a tree +- [x] Insert a node at the top of a subtree - [x] Record very best individual in each population, and return at end. - [x] Write our own tree copy operation; deepcopy() is the slowest operation by far. - [x] Hyperparameter tune diff --git a/eureqa.jl b/eureqa.jl index 88ecc26d..a33fe6be 100644 --- a/eureqa.jl +++ b/eureqa.jl @@ -3,6 +3,21 @@ import Optim const maxdegree = 2 const actualMaxsize = maxsize + maxdegree + +# Sum of square error between two arrays +function SSE(x::Array{Float32}, y::Array{Float32})::Float32 + diff = (x - y) + return sum(diff .* diff) +end + +# Mean of square error between two arrays +function MSE(x::Array{Float32}, y::Array{Float32})::Float32 + return SSE(x, y)/size(x)[1] +end + +const len = size(X)[1] +const baselineMSE = MSE(y, convert(Array{Float32, 1}, ones(len) .* sum(y)/len)) + id = (x,) -> x const nuna = size(unaops)[1] const nbin = size(binops)[1] @@ -211,7 +226,6 @@ end # Evaluate an equation over an array of datapoints function evalTreeArray(tree::Node)::Array{Float32, 1} - len = size(X)[1] if tree.degree == 0 if tree.constant return ones(Float32, len) .* tree.val @@ -225,17 +239,6 @@ function evalTreeArray(tree::Node)::Array{Float32, 1} end end -# Sum of square error between two arrays -function SSE(x::Array{Float32}, y::Array{Float32})::Float32 - diff = (x - y) - return sum(diff .* diff) -end - -# Mean of square error between two arrays -function MSE(x::Array{Float32}, y::Array{Float32})::Float32 - return SSE(x, y)/size(x)[1] -end - # Score an equation function scoreFunc(tree::Node)::Float32 try @@ -608,7 +611,7 @@ function fullRun(niterations::Integer; debug(verbosity, "-----------------------------------------") debug(verbosity, "Complexity \t MSE \t Equation") println(io,"Complexity|MSE|Equation") - for size=1:maxsize + for size=1:actualMaxsize if hallOfFame.exists[size] member = hallOfFame.members[size] numberSmallerAndBetter = sum([member.score > hallOfFame.members[i].score for i=1:(size-1)]) From b408888c7ef36f73b68dbcc3dbed0636d6717281 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 15:32:04 -0400 Subject: [PATCH 02/13] Add insertion op --- eureqa.jl | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/eureqa.jl b/eureqa.jl index a33fe6be..8f49e68d 100644 --- a/eureqa.jl +++ b/eureqa.jl @@ -293,17 +293,80 @@ function appendRandomOp(tree::Node)::Node return tree end -# Select a random node, and replace it an the subtree -# with a variable or constant -function deleteRandomOp(tree::Node)::Node +# Add random node to the top of a tree +function popRandomOp(tree::Node)::Node + node = tree + choice = rand() + makeNewBinOp = choice < nbin/nops + left = tree + + if makeNewBinOp + right = randomConstantNode() + newnode = Node( + binops[rand(1:length(binops))], + left, + right + ) + else + newnode = Node( + unaops[rand(1:length(unaops))], + left + ) + end + node.l = newnode.l + node.r = newnode.r + node.op = newnode.op + node.degree = newnode.degree + node.val = newnode.val + node.constant = newnode.constant + return node +end + +# Insert random node +function insertRandomOp(tree::Node)::Node node = randomNode(tree) - # Can "delete" variable or constant too + choice = rand() + makeNewBinOp = choice < nbin/nops + left = copyNode(node) + + if makeNewBinOp + right = randomConstantNode() + newnode = Node( + binops[rand(1:length(binops))], + left, + right + ) + else + newnode = Node( + unaops[rand(1:length(unaops))], + left + ) + end + node.l = newnode.l + node.r = newnode.r + node.op = newnode.op + node.degree = newnode.degree + node.val = newnode.val + node.constant = newnode.constant + return tree +end + +function randomConstantNode()::Node if rand() > 0.5 val = Float32(randn()) else val = rand(1:nvar) end newnode = Node(val) + return newnode +end + +# Select a random node, and replace it an the subtree +# with a variable or constant +function deleteRandomOp(tree::Node)::Node + node = randomNode(tree) + # Can "delete" variable or constant too + newnode = randomConstantNode() node.l = newnode.l node.r = newnode.r node.op = newnode.op @@ -357,7 +420,12 @@ function iterate( elseif mutationChoice < cweights[2] tree = mutateOperator(tree) elseif mutationChoice < cweights[3] && n < maxsize - tree = appendRandomOp(tree) + toInsert = rand() < 0.1 + if toInsert + tree = insertRandomOp(tree) + else + tree = appendRandomOp(tree) + end elseif mutationChoice < cweights[4] tree = deleteRandomOp(tree) elseif mutationChoice < cweights[5] From df022a05cf4e648b0055e5c98f66861cd811843a Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 15:42:05 -0400 Subject: [PATCH 03/13] Save all members --- eureqa.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/eureqa.jl b/eureqa.jl index 8f49e68d..1c5b490a 100644 --- a/eureqa.jl +++ b/eureqa.jl @@ -651,6 +651,7 @@ function fullRun(niterations::Integer; # Spawn threads to run indepdent evolutions, then gather them @inbounds Threads.@threads for i=1:nthreads allPops[i] = run(allPops[i], ncyclesperiteration, annealing, verbosity=verbosity) + topn = npop bestSubPops[i] = bestSubPop(allPops[i], topn=topn) for j=1:bestSubPops[i].n bestSubPops[i].members[j].tree = simplifyTree(bestSubPops[i].members[j].tree) From c7ce1c2149992d90575d1ba6346fd334c61681fc Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 16:52:47 -0400 Subject: [PATCH 04/13] Better defaults --- eureqa.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/eureqa.py b/eureqa.py index 85f1d789..415fb0f7 100644 --- a/eureqa.py +++ b/eureqa.py @@ -6,27 +6,25 @@ import pandas as pd # Dumped from hyperparam optimization -default_alpha = 5.429178 -default_annealing = 1.000000 -default_fractionReplaced = 0.415076 -default_fractionReplacedHof = 0.031713 -default_ncyclesperiteration = 1614.000000 -default_niterations = 25.000000 -default_npop = 300.000000 -default_parsimony = 0.001861 -default_topn = 9.000000 -default_weightAddNode = 3.788920 -default_weightDeleteNode = 2.553399 -default_weightDoNothing = 0.498528 -default_weightMutateConstant = 2.081372 -default_weightMutateOperator = 2.003413 -default_weightRandomize = 4.679883 -default_weightSimplify = 0.009620 -default_result = -1.183938 +default_alpha = 3.861222 +default_fractionReplaced = 0.057940 +default_fractionReplacedHof = 0.206182 +default_npop = 124.000000 +default_weightAddNode = 1.599672 +default_weightDeleteNode = 0.049554 +default_weightMutateConstant = 5.295328 +default_weightMutateOperator = 0.465999 +default_weightRandomize = 0.184765 +default_weightSimplify = 0.149432 +default_weightDoNothing = 1.000000 +default_result = 0.028084 +default_topn = 10 +default_parsimony = 1e-3 + def eureqa(X=None, y=None, threads=4, niterations=20, - ncyclesperiteration=int(default_ncyclesperiteration), + ncyclesperiteration=10000, binary_operators=["plus", "mult"], unary_operators=["cos", "exp", "sin"], alpha=default_alpha, @@ -34,7 +32,7 @@ def eureqa(X=None, y=None, threads=4, fractionReplaced=default_fractionReplaced, fractionReplacedHof=default_fractionReplacedHof, npop=int(default_npop), - parsimony=default_parsimony, + parsimony=1e-3, migration=True, hofMigration=True, shouldOptimizeConstants=True, @@ -199,7 +197,7 @@ def eureqa(X=None, y=None, threads=4, parser.add_argument("--maxsize", type=int, default=20, help="Max size of equation") parser.add_argument("--niterations", type=int, default=20, help="Number of total migration periods") parser.add_argument("--npop", type=int, default=int(default_npop), help="Number of members per population") - parser.add_argument("--ncyclesperiteration", type=int, default=int(default_ncyclesperiteration), help="Number of evolutionary cycles per migration") + parser.add_argument("--ncyclesperiteration", type=int, default=10000, help="Number of evolutionary cycles per migration") parser.add_argument("--topn", type=int, default=int(default_topn), help="How many best species to distribute from each population") parser.add_argument("--fractionReplacedHof", type=float, default=default_fractionReplacedHof, help="Fraction of population to replace with hall of fame") parser.add_argument("--fractionReplaced", type=float, default=default_fractionReplaced, help="Fraction of population to replace with best from other populations") From c4d7c3f320b17f7f7c0a2c562ae04dc8b076ac4d Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 19:34:02 -0400 Subject: [PATCH 05/13] Improve hyperopt --- hyperparamopt.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hyperparamopt.py b/hyperparamopt.py index 3fb4741f..a03eb4d2 100644 --- a/hyperparamopt.py +++ b/hyperparamopt.py @@ -37,9 +37,14 @@ def run_trial(args): for key in 'niterations npop'.split(' '): args[key] = int(args[key]) - total_steps = 10*100*5000 + + total_steps = 20*100*5000 niterations = args['niterations'] npop = args['npop'] + if niterations == 0 or npop == 0: + print("Bad parameters") + return {'status': 'ok', 'loss': np.inf} + args['ncyclesperiteration'] = int(total_steps / (niterations * npop)) args['topn'] = 10 args['parsimony'] = 1e-3 @@ -52,7 +57,7 @@ def run_trial(args): args['weightDoNothing'] = 1.0 - maxTime = 2*60 + maxTime = 3*60 ntrials = 2 equation_file = f'.hall_of_fame_{np.random.rand():f}.csv' From 2e104ccb85023a488ba520637d08a18e3f19e931 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 19:53:12 -0400 Subject: [PATCH 06/13] New arg for insert node weight --- eureqa.jl | 15 ++++++--------- eureqa.py | 12 ++++++++++++ hyperparamopt.py | 1 + 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/eureqa.jl b/eureqa.jl index 1c5b490a..270c97fe 100644 --- a/eureqa.jl +++ b/eureqa.jl @@ -420,18 +420,15 @@ function iterate( elseif mutationChoice < cweights[2] tree = mutateOperator(tree) elseif mutationChoice < cweights[3] && n < maxsize - toInsert = rand() < 0.1 - if toInsert - tree = insertRandomOp(tree) - else - tree = appendRandomOp(tree) - end - elseif mutationChoice < cweights[4] - tree = deleteRandomOp(tree) + tree = appendRandomOp(tree) + elseif mutationChoice < cweights[4] && n < maxsize + tree = insertRandomOp(tree) elseif mutationChoice < cweights[5] + tree = deleteRandomOp(tree) + elseif mutationChoice < cweights[6] tree = simplifyTree(tree) # Sometimes we simplify tree return tree - elseif mutationChoice < cweights[6] + elseif mutationChoice < cweights[7] tree = genRandomTree(5) # Sometimes we simplify tree else return tree diff --git a/eureqa.py b/eureqa.py index 415fb0f7..f43c272f 100644 --- a/eureqa.py +++ b/eureqa.py @@ -11,6 +11,7 @@ default_fractionReplacedHof = 0.206182 default_npop = 124.000000 default_weightAddNode = 1.599672 +default_weightInsertNode = 1.599672 default_weightDeleteNode = 0.049554 default_weightMutateConstant = 5.295328 default_weightMutateOperator = 0.465999 @@ -38,6 +39,7 @@ def eureqa(X=None, y=None, threads=4, shouldOptimizeConstants=True, topn=int(default_topn), weightAddNode=default_weightAddNode, + weightInsertNode=default_weightInsertNode, weightDeleteNode=default_weightDeleteNode, weightDoNothing=default_weightDoNothing, weightMutateConstant=default_weightMutateConstant, @@ -82,6 +84,7 @@ def eureqa(X=None, y=None, threads=4, constants (Nelder-Mead/Newton) at the end of each iteration. :param topn: int, How many top individuals migrate from each population. :param weightAddNode: float, Relative likelihood for mutation to add a node + :param weightInsertNode: float, Relative likelihood for mutation to insert a node :param weightDeleteNode: float, Relative likelihood for mutation to delete a node :param weightDoNothing: float, Relative likelihood for mutation to leave the individual :param weightMutateConstant: float, Relative likelihood for mutation to change @@ -139,6 +142,7 @@ def eureqa(X=None, y=None, threads=4, {weightMutateConstant:f}, {weightMutateOperator:f}, {weightAddNode:f}, + {weightInsertNode:f}, {weightDeleteNode:f}, {weightSimplify:f}, {weightRandomize:f}, @@ -201,6 +205,14 @@ def eureqa(X=None, y=None, threads=4, parser.add_argument("--topn", type=int, default=int(default_topn), help="How many best species to distribute from each population") parser.add_argument("--fractionReplacedHof", type=float, default=default_fractionReplacedHof, help="Fraction of population to replace with hall of fame") parser.add_argument("--fractionReplaced", type=float, default=default_fractionReplaced, help="Fraction of population to replace with best from other populations") + parser.add_argument("--weightAddNode", type=float, default=default_weightAddNode) + parser.add_argument("--weightInsertNode", type=float, default=default_weightInsertNode) + parser.add_argument("--weightDeleteNode", type=float, default=default_weightDeleteNode) + parser.add_argument("--weightMutateConstant", type=float, default=default_weightMutateConstant) + parser.add_argument("--weightMutateOperator", type=float, default=default_weightMutateOperator) + parser.add_argument("--weightRandomize", type=float, default=default_weightRandomize) + parser.add_argument("--weightSimplify", type=float, default=default_weightSimplify) + parser.add_argument("--weightDoNothing", type=float, default=default_weightDoNothing) parser.add_argument("--migration", type=bool, default=True, help="Whether to migrate") parser.add_argument("--hofMigration", type=bool, default=True, help="Whether to have hall of fame migration") parser.add_argument("--shouldOptimizeConstants", type=bool, default=True, help="Whether to use classical optimization on constants before every migration (doesn't impact performance that much)") diff --git a/hyperparamopt.py b/hyperparamopt.py index a03eb4d2..e2716d49 100644 --- a/hyperparamopt.py +++ b/hyperparamopt.py @@ -117,6 +117,7 @@ def run_trial(args): 'weightMutateConstant': hp.lognormal('weightMutateConstant', np.log(4.0), 1.0), 'weightMutateOperator': hp.lognormal('weightMutateOperator', np.log(0.5), 1.0), 'weightAddNode': hp.lognormal('weightAddNode', np.log(0.5), 1.0), + 'weightInsertNode': hp.lognormal('weightInsertNode', np.log(0.5), 1.0), 'weightDeleteNode': hp.lognormal('weightDeleteNode', np.log(0.5), 1.0), 'weightSimplify': hp.lognormal('weightSimplify', np.log(0.05), 1.0), 'weightRandomize': hp.lognormal('weightRandomize', np.log(0.25), 1.0), From 08a98f6f1949eaeaeaab9ddcb7d40a44df7eae65 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 22:11:25 -0400 Subject: [PATCH 07/13] Function to return random node AND parent --- eureqa.jl | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/eureqa.jl b/eureqa.jl index 270c97fe..c664c7b7 100644 --- a/eureqa.jl +++ b/eureqa.jl @@ -361,6 +361,31 @@ function randomConstantNode()::Node return newnode end +# Return a random node from the tree with parent +function randomNodeAndParent(tree::Node, parent::Union{Node, Nothing})::Tuple{Node, Union{Node, Nothing}} + if tree.degree == 0 + return tree, parent + end + a = countNodes(tree) + b = 0 + c = 0 + if tree.degree >= 1 + b = countNodes(tree.l) + end + if tree.degree == 2 + c = countNodes(tree.r) + end + + i = rand(1:1+b+c) + if i <= b + return randomNodeAndParent(tree.l, tree) + elseif i == b + 1 + return tree, parent + end + + return randomNodeAndParent(tree.r, tree) +end + # Select a random node, and replace it an the subtree # with a variable or constant function deleteRandomOp(tree::Node)::Node From d1327f1097636570ab30d5c854350f395e4fa918 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 22:12:10 -0400 Subject: [PATCH 08/13] Rewrite delete function to join cutoff subtree with parent --- eureqa.jl | 50 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/eureqa.jl b/eureqa.jl index c664c7b7..6849ce21 100644 --- a/eureqa.jl +++ b/eureqa.jl @@ -389,15 +389,47 @@ end # Select a random node, and replace it an the subtree # with a variable or constant function deleteRandomOp(tree::Node)::Node - node = randomNode(tree) - # Can "delete" variable or constant too - newnode = randomConstantNode() - node.l = newnode.l - node.r = newnode.r - node.op = newnode.op - node.degree = newnode.degree - node.val = newnode.val - node.constant = newnode.constant + node, parent = randomNodeAndParent(tree, nothing) + isroot = (parent == nothing) + + if node.degree == 0 + # Replace with new constant + newnode = randomConstantNode() + node.l = newnode.l + node.r = newnode.r + node.op = newnode.op + node.degree = newnode.degree + node.val = newnode.val + node.constant = newnode.constant + elseif node.degree == 1 + # Join one of the children with the parent + if isroot + return node.l + elseif parent.l == node + parent.l = node.l + else + parent.r = node.l + end + else + # Join one of the children with the parent + if rand() < 0.5 + if isroot + return node.l + elseif parent.l == node + parent.l = node.l + else + parent.r = node.l + end + else + if isroot + return node.r + elseif parent.l == node + parent.l = node.r + else + parent.r = node.r + end + end + end return tree end From 47712b5f36b09473d4fa669601aaa8a6662a3fdc Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 22:12:53 -0400 Subject: [PATCH 09/13] Function to combine redundant constants for plus, multiply --- eureqa.jl | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/eureqa.jl b/eureqa.jl index 6849ce21..e4d6501b 100644 --- a/eureqa.jl +++ b/eureqa.jl @@ -433,6 +433,38 @@ function deleteRandomOp(tree::Node)::Node return tree end +# Simplify tree +function combineOperators(tree::Node)::Node + # (const (+*) const) already accounted for + # ((const + var) + const) => (const + var) + # ((const * var) * const) => (const * var) + # (anything commutative!) + if tree.degree == 2 && (tree.op == plus || tree.op == mult) + op = tree.op + if tree.l.constant || tree.r.constant + # Put the constant in r + if tree.l.constant + tmp = tree.r + tree.r = tree.l + tree.l = tmp + end + topconstant = tree.r.val + # Simplify down first + tree.l = combineOperators(tree.l) + below = tree.l + if below.degree == 2 && below.op == op + if below.l.constant + tree = below + tree.l.val = op(tree.l.val, topconstant) + elseif below.r.constant + tree = below + tree.r.val = op(tree.r.val, topconstant) + end + end + end + end + return tree +end # Simplify tree function simplifyTree(tree::Node)::Node @@ -484,6 +516,7 @@ function iterate( tree = deleteRandomOp(tree) elseif mutationChoice < cweights[6] tree = simplifyTree(tree) # Sometimes we simplify tree + tree = combineOperators(tree) # See if repeated constants at outer levels return tree elseif mutationChoice < cweights[7] tree = genRandomTree(5) # Sometimes we simplify tree From ea6850d00342f6b3c705f79ae58526b7eb7ef234 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 22:13:31 -0400 Subject: [PATCH 10/13] Fix printing of equations to ignore size --- eureqa.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eureqa.jl b/eureqa.jl index e4d6501b..a982c66f 100644 --- a/eureqa.jl +++ b/eureqa.jl @@ -770,11 +770,12 @@ function fullRun(niterations::Integer; for size=1:actualMaxsize if hallOfFame.exists[size] member = hallOfFame.members[size] - numberSmallerAndBetter = sum([member.score > hallOfFame.members[i].score for i=1:(size-1)]) + curMSE = MSE(evalTreeArray(member.tree), y) + numberSmallerAndBetter = sum([curMSE > MSE(evalTreeArray(hallOfFame.members[i].tree), y) for i=1:(size-1)]) betterThanAllSmaller = (numberSmallerAndBetter == 0) if betterThanAllSmaller - debug(verbosity, "$size \t $(member.score-parsimony*size) \t $(stringTree(member.tree))") - println(io, "$size|$(member.score-parsimony*size)|$(stringTree(member.tree))") + debug(verbosity, "$size \t $(curMSE) \t $(stringTree(member.tree))") + println(io, "$size|$(curMSE)|$(stringTree(member.tree))") push!(dominating, member) end end From c4bc4e3f8845d9a120617b345d4c42ed10b17398 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 22:16:49 -0400 Subject: [PATCH 11/13] Make parsimony relative to baseline loss --- eureqa.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eureqa.jl b/eureqa.jl index a982c66f..6fce2aa5 100644 --- a/eureqa.jl +++ b/eureqa.jl @@ -16,7 +16,8 @@ function MSE(x::Array{Float32}, y::Array{Float32})::Float32 end const len = size(X)[1] -const baselineMSE = MSE(y, convert(Array{Float32, 1}, ones(len) .* sum(y)/len)) +const avgy = sum(y)/len +const baselineSSE = SSE(y, convert(Array{Float32, 1}, ones(len) .* avgy)) id = (x,) -> x const nuna = size(unaops)[1] @@ -242,7 +243,7 @@ end # Score an equation function scoreFunc(tree::Node)::Float32 try - return MSE(evalTreeArray(tree), y) + countNodes(tree)*parsimony + return SSE(evalTreeArray(tree), y)/baselineSSE + countNodes(tree)*parsimony catch error if isa(error, DomainError) return 1f9 From dcb0894f2a26c16074b36785dfc02b9d1029476a Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 22:34:29 -0400 Subject: [PATCH 12/13] Update hyperparams and docs to insertionOp --- README.md | 8 ++++---- eureqa.jl | 1 - eureqa.py | 32 ++++++++++++++++---------------- operators.jl | 4 ++-- 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 3ca2511a..6e2cfc7b 100644 --- a/README.md +++ b/README.md @@ -59,11 +59,11 @@ which gives: ### API What follows is the API reference for running the numpy interface. -Note that most parameters here -have been tuned with ~1000 trials over several example -equations, so you likely don't need to tune them yourself. +You likely don't need to tune the hyperparameters yourself, +but if you would like, you can use `hyperopt.py` as an example. However, you should adjust `threads`, `niterations`, -`binary_operators`, `unary_operators`, and `maxsize` to your requirements. +`binary_operators`, `unary_operators`, and `maxsize` +to your requirements. The program will output a pandas DataFrame containing the equations, mean square error, and complexity. It will also dump to a csv diff --git a/eureqa.jl b/eureqa.jl index 6fce2aa5..9c06fddd 100644 --- a/eureqa.jl +++ b/eureqa.jl @@ -739,7 +739,6 @@ function fullRun(niterations::Integer; # Spawn threads to run indepdent evolutions, then gather them @inbounds Threads.@threads for i=1:nthreads allPops[i] = run(allPops[i], ncyclesperiteration, annealing, verbosity=verbosity) - topn = npop bestSubPops[i] = bestSubPop(allPops[i], topn=topn) for j=1:bestSubPops[i].n bestSubPops[i].members[j].tree = simplifyTree(bestSubPops[i].members[j].tree) diff --git a/eureqa.py b/eureqa.py index f43c272f..72068647 100644 --- a/eureqa.py +++ b/eureqa.py @@ -6,21 +6,21 @@ import pandas as pd # Dumped from hyperparam optimization -default_alpha = 3.861222 -default_fractionReplaced = 0.057940 -default_fractionReplacedHof = 0.206182 -default_npop = 124.000000 -default_weightAddNode = 1.599672 -default_weightInsertNode = 1.599672 -default_weightDeleteNode = 0.049554 -default_weightMutateConstant = 5.295328 -default_weightMutateOperator = 0.465999 -default_weightRandomize = 0.184765 -default_weightSimplify = 0.149432 -default_weightDoNothing = 1.000000 -default_result = 0.028084 -default_topn = 10 -default_parsimony = 1e-3 +default_alpha = 5 +default_fractionReplaced = 0.1 +default_fractionReplacedHof = 0.1 +default_npop = 200 +default_weightAddNode = 1 +default_weightInsertNode = 1 +default_weightDeleteNode = 1 +default_weightMutateConstant = 10 +default_weightMutateOperator = 1 +default_weightRandomize = 1 +default_weightSimplify = 1 +default_weightDoNothing = 1 +default_result = 1 +default_topn = 10 +default_parsimony = 0.0 def eureqa(X=None, y=None, threads=4, @@ -33,7 +33,7 @@ def eureqa(X=None, y=None, threads=4, fractionReplaced=default_fractionReplaced, fractionReplacedHof=default_fractionReplacedHof, npop=int(default_npop), - parsimony=1e-3, + parsimony=default_parsimony, migration=True, hofMigration=True, shouldOptimizeConstants=True, diff --git a/operators.jl b/operators.jl index 727170b4..db635b49 100644 --- a/operators.jl +++ b/operators.jl @@ -1,6 +1,6 @@ # Define allowed operators. Any julia operator can also be used. -plus(x::Float32, y::Float32)::Float32 = x+y -mult(x::Float32, y::Float32)::Float32 = x*y +plus(x::Float32, y::Float32)::Float32 = x+y #Do not change the name of this operator. +mult(x::Float32, y::Float32)::Float32 = x*y #Do not change the name of this operator. pow(x::Float32, y::Float32)::Float32 = sign(x)*abs(x)^y div(x::Float32, y::Float32)::Float32 = x/y loga(x::Float32)::Float32 = log(abs(x) + 1) From 9f27d223a110d2da8310331d0bb39d4dbeaa5a11 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 18 Sep 2020 22:43:48 -0400 Subject: [PATCH 13/13] Hyperparams similar to previous found by hyperopt --- eureqa.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eureqa.py b/eureqa.py index 72068647..0f933605 100644 --- a/eureqa.py +++ b/eureqa.py @@ -7,8 +7,8 @@ # Dumped from hyperparam optimization default_alpha = 5 -default_fractionReplaced = 0.1 -default_fractionReplacedHof = 0.1 +default_fractionReplaced = 0.30 +default_fractionReplacedHof = 0.05 default_npop = 200 default_weightAddNode = 1 default_weightInsertNode = 1 @@ -16,7 +16,7 @@ default_weightMutateConstant = 10 default_weightMutateOperator = 1 default_weightRandomize = 1 -default_weightSimplify = 1 +default_weightSimplify = 0.1 default_weightDoNothing = 1 default_result = 1 default_topn = 10