From 90fe5574432468164dcd5955ae113052c190916d Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 1 Nov 2023 14:18:35 -0700 Subject: [PATCH 01/84] Fixed a tyypo --- skema/program_analysis/CAST/matlab/tests/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/utils.py b/skema/program_analysis/CAST/matlab/tests/utils.py index 73a7cf49300..89f6e405624 100644 --- a/skema/program_analysis/CAST/matlab/tests/utils.py +++ b/skema/program_analysis/CAST/matlab/tests/utils.py @@ -29,7 +29,7 @@ def assert_operand(operand, value = ""): assert(False) def assert_assignment(assignment, left = "", right = ""): - """ Test an Assignment correct type and operands. """ + """ Test an Assignment for correct type and operands. """ assert isinstance(assignment, Assignment) assert_operand(assignment.left, left) assert_operand(assignment.right, right) @@ -43,16 +43,13 @@ def assert_expression(expression, op = "", left = "", right = ""): def first_cast_node(source): """ Return the first node from the first Module of MatlabToCast output """ - # there should only be one CAST object in the cast output list cast = MatlabToCast(source = source).out_cast assert len(cast) == 1 - # there should be one module in the CAST object assert len(cast[0].nodes) == 1 module = cast[0].nodes[0] assert isinstance(module, Module) - # currently we support one node per module. This may change assert len(module.body) == 1 return module.body[0] From 206e0bd738a4741bb467e543c2945fec2c28feaa Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 1 Nov 2023 18:32:09 -0700 Subject: [PATCH 02/84] Renamed file --- .../tests/{test_conditional_logic.py => test_conditional.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename skema/program_analysis/CAST/matlab/tests/{test_conditional_logic.py => test_conditional.py} (100%) diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional_logic.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py similarity index 100% rename from skema/program_analysis/CAST/matlab/tests/test_conditional_logic.py rename to skema/program_analysis/CAST/matlab/tests/test_conditional.py From 7e430e5651d7bc1bc1714262e75354f869404136 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Thu, 2 Nov 2023 01:16:10 -0700 Subject: [PATCH 03/84] Tests can now use multiple CAST objects --- .../CAST/matlab/tests/test_assignment.py | 17 ++++++++------ .../matlab/tests/test_binary_operation.py | 19 ++++++++------- .../CAST/matlab/tests/test_conditional.py | 10 ++++---- .../CAST/matlab/tests/test_switch.py | 23 +++++++++++++++++++ .../CAST/matlab/tests/utils.py | 9 ++++---- 5 files changed, 51 insertions(+), 27 deletions(-) create mode 100644 skema/program_analysis/CAST/matlab/tests/test_switch.py diff --git a/skema/program_analysis/CAST/matlab/tests/test_assignment.py b/skema/program_analysis/CAST/matlab/tests/test_assignment.py index 898432720c4..489e549d1c6 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_assignment.py +++ b/skema/program_analysis/CAST/matlab/tests/test_assignment.py @@ -1,6 +1,6 @@ from skema.program_analysis.CAST.matlab.tests.utils import ( assert_assignment, - first_cast_node + cast_nodes ) # Test the CAST returned by processing the simplest MATLAB assignment @@ -8,10 +8,13 @@ def test_assignment(): """ Test CAST from MATLAB 'assignment' statement.""" - source = 'x = 5' + source = """ + x = 5 + y = 4 + """ - # The root of the CAST should be Assignment - assignment = first_cast_node(source) - - # The module body should contain a single assignment node - assert_assignment(assignment, left = "x", right = "5") + # nodes should be two assignments + nodes = cast_nodes(source) + assert len(nodes) == 2 + assert_assignment(nodes[0], left = "x", right = "5") + assert_assignment(nodes[1], left = "y", right = "4") diff --git a/skema/program_analysis/CAST/matlab/tests/test_binary_operation.py b/skema/program_analysis/CAST/matlab/tests/test_binary_operation.py index b0452445992..2496bdae13c 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_binary_operation.py +++ b/skema/program_analysis/CAST/matlab/tests/test_binary_operation.py @@ -1,23 +1,22 @@ from skema.program_analysis.CAST.matlab.tests.utils import ( assert_var, assert_expression, - first_cast_node + cast_nodes ) from skema.program_analysis.CAST2FN.model.cast import Assignment -# Test the CAST returned by processing the simplest MATLAB binary operation - def test_binary_operation(): """ Test CAST from MATLAB binary operation statement.""" source = 'z = x + y' - # The root of the CAST should be Assignment - assignment = first_cast_node(source) - assert isinstance(assignment, Assignment) + # cast nodes should be one assignment + nodes = cast_nodes(source) + assert len(nodes) == 1 + assert isinstance(nodes[0], Assignment) - # Left operand of this assignment node is the variable - assert_var(assignment.left, name = "z") + # Left assignment operand is the variable + assert_var(nodes[0].left, name = "z") - # right operand of this assignment node is a binary expression - assert_expression(assignment.right, op = "+", left = "x", right = "y") + # right assignment operand is a binary expression + assert_expression(nodes[0].right, op = "+", left = "x", right = "y") diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py index 59f4d18f815..dd73a25676f 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_conditional.py +++ b/skema/program_analysis/CAST/matlab/tests/test_conditional.py @@ -1,7 +1,7 @@ from skema.program_analysis.CAST.matlab.tests.utils import ( assert_assignment, assert_expression, - first_cast_node + cast_nodes ) from skema.program_analysis.CAST2FN.model.cast import ModelIf @@ -14,7 +14,8 @@ def test_if(): end """ - mi = first_cast_node(source) + mi = cast_nodes(source)[0] + # if assert isinstance(mi, ModelIf) assert_expression(mi.expr, op = ">", left = "x", right = "5") @@ -31,7 +32,7 @@ def test_if_else(): end """ - mi = first_cast_node(source) + mi = cast_nodes(source)[0] # if assert isinstance(mi, ModelIf) assert_expression(mi.expr, op = ">", left = "x", right = "5") @@ -52,7 +53,7 @@ def test_if_elseif_else(): end """ - mi = first_cast_node(source) + mi = cast_nodes(source)[0] # if assert isinstance(mi, ModelIf) assert_expression(mi.expr, op = ">", left = "x", right = "5") @@ -63,4 +64,3 @@ def test_if_elseif_else(): assert_assignment(mi.orelse[0].body[0], left="y", right = "x") # else assert_assignment(mi.orelse[1], left="y", right = "0") - diff --git a/skema/program_analysis/CAST/matlab/tests/test_switch.py b/skema/program_analysis/CAST/matlab/tests/test_switch.py new file mode 100644 index 00000000000..b1fdb12dda2 --- /dev/null +++ b/skema/program_analysis/CAST/matlab/tests/test_switch.py @@ -0,0 +1,23 @@ +from skema.program_analysis.CAST.matlab.tests.utils import ( + assert_var, + assert_expression, + cast_nodes +) +from skema.program_analysis.CAST2FN.model.cast import Assignment + +def do_not_test_switch(): + """ Test CAST from MATLAB switch statement.""" + + source = """ + switch s + case 'axis' + n = 0; + case 'bottom' + n = 1; + case 'top' + n = 2; + end + """ + + # The root of the CAST should be Assignment + nodes = cast_nodes(source) diff --git a/skema/program_analysis/CAST/matlab/tests/utils.py b/skema/program_analysis/CAST/matlab/tests/utils.py index 89f6e405624..68e1e65b0d1 100644 --- a/skema/program_analysis/CAST/matlab/tests/utils.py +++ b/skema/program_analysis/CAST/matlab/tests/utils.py @@ -41,8 +41,8 @@ def assert_expression(expression, op = "", left = "", right = ""): assert_operand(expression.operands[0], left) assert_operand(expression.operands[1], right) -def first_cast_node(source): - """ Return the first node from the first Module of MatlabToCast output """ +def cast_nodes(source): + """ Return the CAST nodes from the first Module of MatlabToCast output """ # there should only be one CAST object in the cast output list cast = MatlabToCast(source = source).out_cast assert len(cast) == 1 @@ -50,6 +50,5 @@ def first_cast_node(source): assert len(cast[0].nodes) == 1 module = cast[0].nodes[0] assert isinstance(module, Module) - # currently we support one node per module. This may change - assert len(module.body) == 1 - return module.body[0] + # return the module body node list + return module.body From 77fc4c7232daeebe16bd779d798ae0cd28da4057 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Thu, 2 Nov 2023 01:33:34 -0700 Subject: [PATCH 04/84] Added a string coparison to the assignment tests --- skema/program_analysis/CAST/matlab/tests/test_assignment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_assignment.py b/skema/program_analysis/CAST/matlab/tests/test_assignment.py index 489e549d1c6..08fd89af13a 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_assignment.py +++ b/skema/program_analysis/CAST/matlab/tests/test_assignment.py @@ -10,11 +10,13 @@ def test_assignment(): source = """ x = 5 - y = 4 + y = "xxx" """ # nodes should be two assignments nodes = cast_nodes(source) assert len(nodes) == 2 assert_assignment(nodes[0], left = "x", right = "5") - assert_assignment(nodes[1], left = "y", right = "4") + + # When comparing strings you must include escaped quotes + assert_assignment(nodes[1], left = "y", right = "\"xxx\"") From 6095d71562172ad5d396c369c066df5177b38f6b Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Thu, 2 Nov 2023 16:34:30 -0700 Subject: [PATCH 05/84] Cast debug with multiple input files --- skema/program_analysis/CAST/matlab/cast_out | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index 91cb0adcc04..2b27ee6c63d 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -3,24 +3,28 @@ import json import sys from skema.program_analysis.CAST.matlab.matlab_to_cast import MatlabToCast -""" Run a file of any type through the Tree-sitter MATLAB parser""" + +def show_cast(filename): + """ Run a file of any type through the Tree-sitter MATLAB parser""" + parser = MatlabToCast(filename) + print("\n\nINPUT:") + print(parser.filename) + print("\nSOURCE:") + print(parser.source) + print('\nCAST:') + cast_list = parser.out_cast + for cast in cast_list: + jd = json.dumps( + cast.to_json_object(), + sort_keys=True, + indent=2, + ) + print(jd) if __name__ == "__main__": if len(sys.argv) > 1: for i in range(1, len(sys.argv)): - parser = MatlabToCast(sys.argv[i]) - print("\n\nINPUT:") - print(parser.filename) - print("\nSOURCE:") - print(parser.source) - print('\nCAST:') - cast_list = parser.out_cast - for cast_index in range(0, len(cast_list)): - jd = json.dumps( - cast_list[cast_index].to_json_object(), - sort_keys=True, - indent=2, - ) - print(jd) + show_cast(sys.argv[i]) else: print("Please enter one filename to parse") + From 9d2425714d363587072eebcf0a9c9d4b06a8c468 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Fri, 3 Nov 2023 12:03:54 -0700 Subject: [PATCH 06/84] Added a method for pruning keys from cast debug output --- skema/program_analysis/CAST/matlab/cast_out | 30 ++++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index 2b27ee6c63d..806304622f7 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -3,23 +3,39 @@ import json import sys from skema.program_analysis.CAST.matlab.matlab_to_cast import MatlabToCast +def prune(json_obj, target_key): + """ remove all instances of the target key from the json object""" + def remove_key(json_obj, target_key): + if isinstance(json_obj, dict): + if target_key in json_obj.keys(): + json_obj.pop(target_key) + for key in json_obj.keys(): + remove_key(json_obj[key], target_key) + elif isinstance(json_obj, list): + for item in json_obj: + remove_key(item, target_key) + return(json_obj) + + print(f"Removed key: {target_key}") + return remove_key(json_obj, target_key) def show_cast(filename): """ Run a file of any type through the Tree-sitter MATLAB parser""" parser = MatlabToCast(filename) - print("\n\nINPUT:") + print("\nINPUT FILE:") print(parser.filename) print("\nSOURCE:") print(parser.source) print('\nCAST:') cast_list = parser.out_cast + for cast in cast_list: - jd = json.dumps( - cast.to_json_object(), - sort_keys=True, - indent=2, - ) - print(jd) + json_obj = cast.to_json_object() + + # declutter + json_obj = prune(json_obj, "source_refs") + + print(json.dumps(json_obj, sort_keys = True, indent = 2)) if __name__ == "__main__": if len(sys.argv) > 1: From 53fe75668b773076b08923cc2a4caf6ebb321150 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Fri, 3 Nov 2023 14:11:32 -0700 Subject: [PATCH 07/84] early switch statment handler --- .../CAST/matlab/matlab_to_cast.py | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index d29bc8bfeb0..8e13d3e7220 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -78,6 +78,7 @@ def generate_cast(self) -> List[CAST]: """Interface for generating CAST.""" # remove comments from tree before processing + modules = self.run(remove_comments(self.tree.root_node)) return [CAST([module], "matlab") for module in modules] @@ -96,7 +97,6 @@ def run_old(self, root) -> List[Module]: "function_definition", "subroutine", "assignment", - "switch_statement" ] outer_body_nodes = get_children_by_types(root, body_node_names) @@ -152,6 +152,10 @@ def visit(self, node): return self.visit_extent_specifier(node) elif node.type == "do_loop_statement": return self.visit_do_loop_statement(node) + elif node.type == "switch_statement": + return self.visit_switch_statement(node) + elif node.type == "case_clause": + return self.visit_case_clause(node) elif node.type == "if_statement": return self.visit_if_statement(node) elif node.type == "elseif_clause": @@ -578,6 +582,45 @@ def visit_do_loop_statement(self, node) -> Loop: source_refs=[self.node_helper.get_source_ref(node)], ) + + def visit_switch_statement(self, node): + """ return a conditional logic statement based on the case statement """ + # This case statement: + # switch s + # case 'top' + # n = 1; + # case 'bottom' + # n = 0; + # end + # + # can be reduced to: + # if s == ‘top’ + # n = 1 + # elseif s == ‘bottom’ + # n = 2 + # + # The first case clause becomes an if statement + # any subsequent case clauses become elseif statements + # a default clause, if one exists, becomes an else statement + + mi = ModelIf() + + # get switch identifier + identifier = get_first_child_by_type(node, ("identifier")) + + # get n case clauses + types = list() + types.append("case_clause") + case_clauses = get_children_by_types(node, types) + + return mi + + + def visit_case_clause(self, node): + """ return an if statement defining the case clause""" + pass + + def visit_if_statement(self, node): """ return a ModelIf if, elseif, and else clauses""" @@ -621,8 +664,8 @@ def visit_elseif_clause(self, node): # get ModelIf with body nodes mi = self.visit_else_clause(node) # addd comparison operator - comp: Operator = get_first_child_by_type(node, "comparison_operator") - mi.expr = self.visit(comp) + comparison_operator = get_first_child_by_type(node, "comparison_operator") + mi.expr = self.visit(comparison_operator) return mi From 3502db1de19a3ce15b83d12145da59c3f92e9b66 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Fri, 3 Nov 2023 14:35:16 -0700 Subject: [PATCH 08/84] Fixed node_helper args --- skema/program_analysis/CAST/matlab/matlab_to_cast.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 8e13d3e7220..6b35b868930 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -606,12 +606,10 @@ def visit_switch_statement(self, node): mi = ModelIf() # get switch identifier - identifier = get_first_child_by_type(node, ("identifier")) + identifier = get_first_child_by_type(node, "identifier") # get n case clauses - types = list() - types.append("case_clause") - case_clauses = get_children_by_types(node, types) + case_clauses = get_children_by_types(node, ["case_clause"]) return mi @@ -638,7 +636,7 @@ def visit_if_statement(self, node): mi = self.visit_elseif_clause(node) # get 0-n elseif_clauses - elseif_clauses = get_children_by_types(node, ("elseif_clause")) + elseif_clauses = get_children_by_types(node, ["elseif_clause"]) for child in elseif_clauses: elseif_node = self.visit(child) if elseif_node: @@ -647,7 +645,7 @@ def visit_if_statement(self, node): mi.orelse.append(elseif_node) # get 0-1 else_clauses - else_clauses = get_children_by_types(node, ("else_clause")) + else_clauses = get_children_by_types(node, ["else_clause"]) for child in else_clauses: else_node = self.visit(child) if else_node.body: From 9ef903a0fb4f91e4cc9cb4d321f0a7ea7bac1815 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Fri, 3 Nov 2023 15:22:36 -0700 Subject: [PATCH 09/84] cast_out debug with multiple key removal --- skema/program_analysis/CAST/matlab/cast_out | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index 806304622f7..f829467b330 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -1,23 +1,25 @@ #!/usr/bin/env python3 import json +from typing import List import sys from skema.program_analysis.CAST.matlab.matlab_to_cast import MatlabToCast -def prune(json_obj, target_key): - """ remove all instances of the target key from the json object""" +def prune(json_obj, target_keys: List): + """ remove all instances of the target keys from the json object""" def remove_key(json_obj, target_key): if isinstance(json_obj, dict): - if target_key in json_obj.keys(): - json_obj.pop(target_key) - for key in json_obj.keys(): - remove_key(json_obj[key], target_key) + for target_key in target_keys: + if target_key in json_obj.keys(): + json_obj.pop(target_key) + for key in json_obj.keys(): + remove_key(json_obj[key], target_keys) elif isinstance(json_obj, list): for item in json_obj: - remove_key(item, target_key) + remove_key(item, target_keys) return(json_obj) - print(f"Removed key: {target_key}") - return remove_key(json_obj, target_key) + print(f"Removed keys: {target_keys}") + return remove_key(json_obj, target_keys) def show_cast(filename): """ Run a file of any type through the Tree-sitter MATLAB parser""" @@ -33,7 +35,7 @@ def show_cast(filename): json_obj = cast.to_json_object() # declutter - json_obj = prune(json_obj, "source_refs") + json_obj = prune(json_obj, ["source_refs"]) print(json.dumps(json_obj, sort_keys = True, indent = 2)) From 88d63eb37b9d1b4b000a45516af06e81a3841504 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Fri, 3 Nov 2023 22:13:31 -0700 Subject: [PATCH 10/84] switch 'if' expression done --- .../CAST/matlab/matlab_to_cast.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 6b35b868930..bb46dee2121 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -154,8 +154,6 @@ def visit(self, node): return self.visit_do_loop_statement(node) elif node.type == "switch_statement": return self.visit_switch_statement(node) - elif node.type == "case_clause": - return self.visit_case_clause(node) elif node.type == "if_statement": return self.visit_if_statement(node) elif node.type == "elseif_clause": @@ -603,20 +601,37 @@ def visit_switch_statement(self, node): # any subsequent case clauses become elseif statements # a default clause, if one exists, becomes an else statement - mi = ModelIf() + def if_clause(identifier, case_clause): + # { + # "body": null, + # "expr": null, + # "node_type": "ModelIf", + # "orelse": null + # } + + operand1 = self.visit(get_first_child_by_type(case_clause, "string")) + + expr = Operator( + op = "==", + operands = [identifier, operand1] + ) + mi = ModelIf(expr = expr) + + return mi + # get switch identifier - identifier = get_first_child_by_type(node, "identifier") + identifier = self.visit(get_first_child_by_type(node, "identifier")) # get n case clauses case_clauses = get_children_by_types(node, ["case_clause"]) - return mi + # use first case clause to create an if statment + if len(case_clauses) > 0: + mi = if_clause(identifier, case_clauses[0]) + return mi - def visit_case_clause(self, node): - """ return an if statement defining the case clause""" - pass def visit_if_statement(self, node): From ce687b84a2a77004d9aa43f6f0820ef656ced1aa Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Sat, 4 Nov 2023 20:45:40 -0700 Subject: [PATCH 11/84] Added key filtering and correct reporting of None values --- skema/program_analysis/CAST/matlab/cast_out | 25 +++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index f829467b330..37dada86696 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -1,10 +1,19 @@ #!/usr/bin/env python3 import json -from typing import List import sys from skema.program_analysis.CAST.matlab.matlab_to_cast import MatlabToCast +from typing import List + +# Show a CAST object as JSON with certain keys filtered for clarity. -def prune(json_obj, target_keys: List): +# Keys to remove from the output for clarity. +KEY_FILTER = [ + "source_refs", + "default_value", + "interpreter" +] + +def remove_keys(json_obj, target_keys: List): """ remove all instances of the target keys from the json object""" def remove_key(json_obj, target_key): if isinstance(json_obj, dict): @@ -33,11 +42,13 @@ def show_cast(filename): for cast in cast_list: json_obj = cast.to_json_object() - - # declutter - json_obj = prune(json_obj, ["source_refs"]) - - print(json.dumps(json_obj, sort_keys = True, indent = 2)) + # declutter JSON by filtering keys + json_obj = remove_keys(json_obj, KEY_FILTER) + # pretty print JSON to string + output = json.dumps(json_obj, sort_keys=True, indent=2) + # json.dumps incorrectly reports None as null. Python has no null. + output = output.replace (": null", ": None") + print(output) if __name__ == "__main__": if len(sys.argv) > 1: From ce2337f51c26389cf137dde0ef015564ab88ae56 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Sat, 4 Nov 2023 20:47:16 -0700 Subject: [PATCH 12/84] Added an 'otherwise' clause to the MATLAB switch statement test --- skema/program_analysis/CAST/matlab/tests/test_switch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_switch.py b/skema/program_analysis/CAST/matlab/tests/test_switch.py index b1fdb12dda2..34b51380226 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_switch.py +++ b/skema/program_analysis/CAST/matlab/tests/test_switch.py @@ -10,12 +10,12 @@ def do_not_test_switch(): source = """ switch s - case 'axis' - n = 0; - case 'bottom' + case 'one' n = 1; - case 'top' + case 'two' n = 2; + otherwise + n = 0; end """ From c8f45f521757cf2cd4b060386464c8160b42673e Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Sat, 4 Nov 2023 20:48:31 -0700 Subject: [PATCH 13/84] Populated Operator fields, fixed comments --- .../CAST/matlab/matlab_to_cast.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index bb46dee2121..6c1df8ef272 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -583,19 +583,23 @@ def visit_do_loop_statement(self, node) -> Loop: def visit_switch_statement(self, node): """ return a conditional logic statement based on the case statement """ - # This case statement: + # This MATLAB case statement: # switch s - # case 'top' + # case 'one' # n = 1; - # case 'bottom' + # case 'two' + # n = 2; + # otherwise # n = 0; # end # # can be reduced to: - # if s == ‘top’ + # if s == 'one' # n = 1 - # elseif s == ‘bottom’ + # elseif s == 'two' # n = 2 + # else + # n = 0 # # The first case clause becomes an if statement # any subsequent case clauses become elseif statements @@ -611,11 +615,13 @@ def if_clause(identifier, case_clause): operand1 = self.visit(get_first_child_by_type(case_clause, "string")) - expr = Operator( + mi = self.visit_else_clause(case_clause) + mi.expr = Operator( op = "==", - operands = [identifier, operand1] + operands = [identifier, operand1], + source_language = "MATLAB", + version = MATLAB_VERSION ) - mi = ModelIf(expr = expr) return mi From 4b23e900d67703ce7fa66f28fa87ddc177e07191 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 11:06:34 -0700 Subject: [PATCH 14/84] Cleaned up code formatting, comments --- skema/program_analysis/CAST/matlab/cast_out | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index 37dada86696..5c196b19073 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -4,18 +4,17 @@ import sys from skema.program_analysis.CAST.matlab.matlab_to_cast import MatlabToCast from typing import List -# Show a CAST object as JSON with certain keys filtered for clarity. +# Show a CAST object as pretty printed JSON with keys filtered for clarity. -# Keys to remove from the output for clarity. -KEY_FILTER = [ - "source_refs", - "default_value", - "interpreter" -] +# Keys to remove from the output +#KEY_FILTER = ["source_refs", "default_value", "interpreter"] +KEY_FILTER = ["source_refs"] def remove_keys(json_obj, target_keys: List): """ remove all instances of the target keys from the json object""" + def remove_key(json_obj, target_key): + """ remove all instances of the target key from the json object""" if isinstance(json_obj, dict): for target_key in target_keys: if target_key in json_obj.keys(): @@ -30,6 +29,7 @@ def remove_keys(json_obj, target_keys: List): print(f"Removed keys: {target_keys}") return remove_key(json_obj, target_keys) + def show_cast(filename): """ Run a file of any type through the Tree-sitter MATLAB parser""" parser = MatlabToCast(filename) @@ -39,21 +39,18 @@ def show_cast(filename): print(parser.source) print('\nCAST:') cast_list = parser.out_cast - for cast in cast_list: json_obj = cast.to_json_object() # declutter JSON by filtering keys json_obj = remove_keys(json_obj, KEY_FILTER) # pretty print JSON to string output = json.dumps(json_obj, sort_keys=True, indent=2) - # json.dumps incorrectly reports None as null. Python has no null. - output = output.replace (": null", ": None") print(output) + if __name__ == "__main__": if len(sys.argv) > 1: for i in range(1, len(sys.argv)): show_cast(sys.argv[i]) else: print("Please enter one filename to parse") - From dc0d015faf10459167f6983589ba6a68be911ae3 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 11:07:09 -0700 Subject: [PATCH 15/84] Cleaned up comments --- .../CAST/matlab/matlab_to_cast.py | 64 ++++++------------- 1 file changed, 18 insertions(+), 46 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 6c1df8ef272..943271e6096 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -158,7 +158,7 @@ def visit(self, node): return self.visit_if_statement(node) elif node.type == "elseif_clause": return self.visit_elseif_clause(node) - elif node.type == "else_clause": + elif node.type in ["else_clause", "otherwise_clause"]: return self.visit_else_clause(node) elif node.type == "derived_type_definition": return self.visit_derived_type(node) @@ -580,71 +580,47 @@ def visit_do_loop_statement(self, node) -> Loop: source_refs=[self.node_helper.get_source_ref(node)], ) + def visit_otherwise_clause(self, node): + mi = ModelIf() + + # ... + + return mi def visit_switch_statement(self, node): - """ return a conditional logic statement based on the case statement """ - # This MATLAB case statement: - # switch s + """ return a conditional logic statement based on the switch statement """ # case 'one' # n = 1; - # case 'two' - # n = 2; + # case {2, '2', 'two'} + # n = 2 # otherwise # n = 0; # end # - # can be reduced to: - # if s == 'one' + # The above statement can be reduced to: + # if s in ['one'] # n = 1 - # elseif s == 'two' + # elseif s in [2, '2', 'two'] # n = 2 - # else + # else # n = 0 - # # The first case clause becomes an if statement # any subsequent case clauses become elseif statements # a default clause, if one exists, becomes an else statement - def if_clause(identifier, case_clause): - # { - # "body": null, - # "expr": null, - # "node_type": "ModelIf", - # "orelse": null - # } - - operand1 = self.visit(get_first_child_by_type(case_clause, "string")) - - mi = self.visit_else_clause(case_clause) - mi.expr = Operator( - op = "==", - operands = [identifier, operand1], - source_language = "MATLAB", - version = MATLAB_VERSION - ) - - return mi - - # get switch identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) # get n case clauses case_clauses = get_children_by_types(node, ["case_clause"]) - # use first case clause to create an if statment - if len(case_clauses) > 0: - mi = if_clause(identifier, case_clauses[0]) + mi = self.visit(get_first_child_by_type(node, "otherwise_clause")) return mi - - def visit_if_statement(self, node): """ return a ModelIf if, elseif, and else clauses""" - # print('visit_if_statement') - # if_statement Tree-sitter syntax tree: # if # comparison_operator @@ -677,17 +653,13 @@ def visit_if_statement(self, node): return mi - def visit_elseif_clause(self, node): - """ return a ModelIf with comparison and body nodes. """ - # get ModelIf with body nodes + """ return a ModelIf with comparison operator and body nodes. """ mi = self.visit_else_clause(node) - # addd comparison operator comparison_operator = get_first_child_by_type(node, "comparison_operator") mi.expr = self.visit(comparison_operator) return mi - def visit_else_clause(self, node): """ Return a ModelIf with body nodes only. """ @@ -703,7 +675,6 @@ def visit_else_clause(self, node): return mi - def visit_assignment(self, node): """Docstring""" # print('visit_assignment') @@ -773,7 +744,6 @@ def visit_literal(self, node) -> LiteralValue: source_refs=[literal_source_ref], ) - def visit_identifier(self, node): """Docstring""" # print('visit_identifier') @@ -1223,3 +1193,5 @@ def get_gromet_function_node(self, func_name: str) -> Name: return self.variable_context.get_node(func_name) return self.variable_context.add_variable(func_name, "function", None) + + From 91d1a02ee30f7a9aae2d7dd89acfc24a12fd2c91 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 11:37:04 -0700 Subject: [PATCH 16/84] Removed debug print statements and placeholder Docstrings --- .../CAST/matlab/matlab_to_cast.py | 61 ++++--------------- .../CAST/matlab/node_helper.py | 3 - .../CAST/matlab/variable_context.py | 8 --- 3 files changed, 12 insertions(+), 60 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 943271e6096..c0dcff05ea8 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -46,7 +46,6 @@ class MatlabToCast(object): def __init__(self, source_path = "", source = ""): - """docstring""" # if a source file path is provided, read source from file if not source_path == "": @@ -118,7 +117,7 @@ def run_old(self, root) -> List[Module]: def visit(self, node): """Switch execution based on node type""" - # print(f"\nvisit node type = {node.type}") + # print(f"\nvisit {node.type}") if node.type in ["program", "module", "source_file"] : return self.visit_module(node) @@ -169,7 +168,6 @@ def visit(self, node): def visit_module(self, node: Node) -> Module: """Visitor for program and module statement. Returns a Module object""" - # print('visit_module') self.variable_context.push_context() program_body = [] @@ -190,13 +188,10 @@ def visit_module(self, node: Node) -> Module: def visit_internal_procedures(self, node: Node) -> List[FunctionDef]: """Visitor for internal procedures. Returns list of FunctionDef""" - # print('visit_internal_procedures') internal_procedures = get_children_by_types(node, ["function_definition", "subroutine"]) return [self.visit(procedure) for procedure in internal_procedures] def visit_name(self, node): - """Docstring""" - # print('visit_name') # Node structure # (name) @@ -210,8 +205,6 @@ def visit_name(self, node): ) def visit_function_def(self, node): - """Docstring""" - # print('visit_function_def') # TODO: Refactor function def code to use new helper functions # Node structure # (subroutine) @@ -318,8 +311,6 @@ def visit_function_def(self, node): ) def visit_function_call(self, node): - """Docstring""" - # print('visit_function_call') # Pull relevent nodes if node.type == "subroutine_call": function_node = node.children[1] @@ -361,8 +352,6 @@ def visit_function_call(self, node): ) def visit_keyword_statement(self, node): - """Docstring""" - # print('visit_keyword_statement') # Currently, the only keyword_identifier produced by tree-sitter is Return # However, there may be other instances @@ -398,8 +387,6 @@ def visit_keyword_statement(self, node): ) def visit_use_statement(self, node): - """Docstring""" - # print('visit_use_statement') # (use) # (use) # (module_name) @@ -459,7 +446,6 @@ def visit_do_loop_statement(self, node) -> Loop: (body) ... """ - # print('visit_do_loop_statement') # First check for # TODO: Add do until Loop support while_statement_node = get_first_child_by_type(node, "while_statement") @@ -661,23 +647,13 @@ def visit_elseif_clause(self, node): return mi - def visit_else_clause(self, node): - """ Return a ModelIf with body nodes only. """ - # get the top level body nodes + def visit_else_clause(self, node: Node): + """ Return a ModelIf with body nodes from Tree-sitter block nodes. """ mi = ModelIf() - block = get_first_child_by_type(node, "block") - for child in block.children: - body_node = self.visit(child) - if body_node: - if not mi.body: - mi.body = list() - mi.body.append(body_node) - + mi.body = self.get_body_nodes(node) return mi def visit_assignment(self, node): - """Docstring""" - # print('visit_assignment') left, _, right = node.children return Assignment( @@ -688,7 +664,6 @@ def visit_assignment(self, node): def visit_literal(self, node) -> LiteralValue: """Visitor for literals. Returns a LiteralValue""" - # print('visit_literal') literal_type = node.type literal_value = self.node_helper.get_identifier(node) literal_source_ref = self.node_helper.get_source_ref(node) @@ -745,8 +720,6 @@ def visit_literal(self, node) -> LiteralValue: ) def visit_identifier(self, node): - """Docstring""" - # print('visit_identifier') # By default, this is unknown, but can be updated by other visitors identifier = self.node_helper.get_identifier(node) if self.variable_context.is_variable(identifier): @@ -769,8 +742,6 @@ def visit_identifier(self, node): ) def visit_math_expression(self, node): - """Docstring""" - # print('visit_math_expression') op = self.node_helper.get_identifier( get_control_children(node)[0] ) # The operator will be the first control character @@ -896,8 +867,6 @@ def visit_variable_declaration(self, node) -> List: return vars def visit_extent_specifier(self, node): - """Docstring""" - # print('visit_extent_specifier') # Node structure # (extent_specifier) # (identifier) @@ -938,8 +907,6 @@ def visit_derived_type(self, node: Node) -> RecordDef: (BODY_NODES) ... """ - # print('visit_derived_type') - record_name = self.node_helper.get_identifier( get_first_child_by_type(node, "type_name", recurse=True) @@ -1021,7 +988,6 @@ def visit_derived_type_member_expression(self, node) -> Attribute: (argument_list) (type_member) """ - # print('visit_derived_type_member_expression') # If we are accessing an attribute of a scalar type, we can simply pull the name node from the variable context. # However, if this is a dimensional type, we must convert it to a call to _get. @@ -1048,8 +1014,6 @@ def visit_derived_type_member_expression(self, node) -> Attribute: # NOTE: This function starts with _ because it will never be dispatched to directly. There is not a get node in the tree-sitter parse tree. # From context, we will determine when we are accessing an element of a List, and call this function, def _visit_get(self, node): - """Docstring""" - # print('_visit_get') # Node structure # (call_expression) # (identifier) @@ -1088,8 +1052,6 @@ def _visit_get(self, node): ) def _visit_set(self, node): - """Docstring""" - # print('_visit_set') # Node structure # (assignment) # (call_expression) @@ -1116,7 +1078,6 @@ def _visit_while(self, node) -> Loop: (...) ... (body) ... """ - # print('_visit_while') while_statement_node = get_first_child_by_type(node, "while_statement") # The first body node will be the node after the while_statement @@ -1145,7 +1106,6 @@ def _visit_while(self, node) -> Loop: def _visit_implied_do_loop(self, node) -> Call: """Custom visitor for implied_do_loop array literal. This form gets converted to a call to range""" # TODO: This loop_control is the same as the do loop. Can we turn this into one visitor? - # print('_visit_implied_do_loop') loop_control_node = get_first_child_by_type( node, "loop_control_expression", recurse=True ) @@ -1174,8 +1134,6 @@ def _visit_implied_do_loop(self, node) -> Call: ) def _visit_passthrough(self, node): - """Docstring""" - # print('_visit_passthrough') if len(node.children) == 0: return None @@ -1185,8 +1143,6 @@ def _visit_passthrough(self, node): return child_cast def get_gromet_function_node(self, func_name: str) -> Name: - """Docstring""" - # print('get_gromet_function_node') # Idealy, we would be able to create a dummy node and just call the name visitor. # However, tree-sitter does not allow you to create or modify nodes, so we have to recreate the logic here. if self.variable_context.is_variable(func_name): @@ -1194,4 +1150,11 @@ def get_gromet_function_node(self, func_name: str) -> Name: return self.variable_context.add_variable(func_name, "function", None) - + def get_body_nodes(self, node): + """ Return valid body nodes from Tree-sitter block node. """ + block = get_first_child_by_type(node, "block") + ast_nodes = [self.visit(child) for child in block.children] + body_nodes = [ast_node for ast_node in ast_nodes if ast_node] + if len(body_nodes) > 0: + return body_nodes + return None diff --git a/skema/program_analysis/CAST/matlab/node_helper.py b/skema/program_analysis/CAST/matlab/node_helper.py index a64af619599..a76a5e91706 100644 --- a/skema/program_analysis/CAST/matlab/node_helper.py +++ b/skema/program_analysis/CAST/matlab/node_helper.py @@ -27,7 +27,6 @@ class NodeHelper(): def __init__(self, source: str, source_file_name: str): - """Docstring""" self.source = source self.source_file_name = source_file_name @@ -114,12 +113,10 @@ def get_last_child_index(node, type: str): def get_control_children(node: Node): - """Docstring""" return get_children_by_types(node, CONTROL_CHARACTERS) def get_non_control_children(node: Node): - """Docstring""" children = [] for child in node.children: if child.type not in CONTROL_CHARACTERS: diff --git a/skema/program_analysis/CAST/matlab/variable_context.py b/skema/program_analysis/CAST/matlab/variable_context.py index 6ef6088964a..b2ab8af51b9 100644 --- a/skema/program_analysis/CAST/matlab/variable_context.py +++ b/skema/program_analysis/CAST/matlab/variable_context.py @@ -6,7 +6,6 @@ class VariableContext(object): def __init__(self): - """Docstring""" self.context = [{}] # Stack of context dictionaries self.context_return_values = [set()] # Stack of context return values self.all_symbols = {} @@ -78,11 +77,9 @@ def is_variable(self, symbol: str) -> bool: return symbol in self.all_symbols def get_node(self, symbol: str) -> Dict: - """Docstring""" return self.all_symbols[symbol]["node"] def get_type(self, symbol: str) -> str: - """Docstring""" return self.all_symbols[symbol]["type"] def update_type(self, symbol: str, type: str): @@ -92,22 +89,18 @@ def update_type(self, symbol: str, type: str): self.all_symbols[full_symbol_name]["type"] = type def add_return_value(self, symbol): - """Docstring""" self.context_return_values[-1].add(symbol) def remove_return_value(self, symbol): - """Docstring""" self.context_return_values[-1].discard(symbol) def generate_iterator(self): - """Docstring""" symbol = f"generated_iter_{self.iterator_id}" self.iterator_id += 1 return self.add_variable(symbol, "iterator", None) def generate_stop_condition(self): - """Docstring""" symbol = f"sc_{self.stop_condition_id}" self.stop_condition_id += 1 @@ -126,5 +119,4 @@ def set_internal(self): self.internal = True def unset_internal(self): - """Docstring""" self.internal = False From 313d6a28897cbcfcf9c1063cd2d17b397df488cb Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 14:22:19 -0700 Subject: [PATCH 17/84] Deleted an unused debug tree field --- skema/program_analysis/CAST/matlab/tree_out | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tree_out b/skema/program_analysis/CAST/matlab/tree_out index 4e52a78f6a7..c0352e3077a 100755 --- a/skema/program_analysis/CAST/matlab/tree_out +++ b/skema/program_analysis/CAST/matlab/tree_out @@ -11,10 +11,7 @@ from skema.program_analysis.tree_sitter_parsers.build_parsers import ( def print_tree(node: Node, indent = ''): """Display the node branch in pretty format""" for child in node.children: - if child.type == "\n": - # print(f"{indent} ") - pass - else: + if not child.type == "\n": print(f"{indent} {child.type}") print_tree(child, indent + ' ') From 6ed19066846c08ed089dd65944634ffab3c8baed Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 17:17:26 -0700 Subject: [PATCH 18/84] Fixed an indentation error --- skema/program_analysis/CAST/matlab/cast_out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index 5c196b19073..2d332dac26b 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -14,7 +14,7 @@ def remove_keys(json_obj, target_keys: List): """ remove all instances of the target keys from the json object""" def remove_key(json_obj, target_key): - """ remove all instances of the target key from the json object""" + """ remove all instances of the target key from the json object""" if isinstance(json_obj, dict): for target_key in target_keys: if target_key in json_obj.keys(): From 5511e15e438197f2798c9bf941eac0169383094f Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 18:55:03 -0700 Subject: [PATCH 19/84] Added test cases for multi-element conditional blocks --- skema/program_analysis/CAST/matlab/tests/test_conditional.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py index dd73a25676f..63183c52824 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_conditional.py +++ b/skema/program_analysis/CAST/matlab/tests/test_conditional.py @@ -27,8 +27,10 @@ def test_if_else(): source = """ if x > 5 y = 6 + three = 3 else y = x + foo = 'bar' end """ @@ -37,8 +39,10 @@ def test_if_else(): assert isinstance(mi, ModelIf) assert_expression(mi.expr, op = ">", left = "x", right = "5") assert_assignment(mi.body[0], left="y", right = "6") + assert_assignment(mi.body[1], left="three", right = "3") # else assert_assignment(mi.orelse[0], left="y", right = "x") + assert_assignment(mi.orelse[1], left="foo", right = "'bar'") def test_if_elseif_else(): """ Test CAST from MATLAB 'if elseif else' conditional logic.""" From 41f89b56eb15cb063a6a510a5da2b02446569ead Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 18:58:44 -0700 Subject: [PATCH 20/84] cleaned up conditional node handler --- .../CAST/matlab/matlab_to_cast.py | 84 ++++++++----------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index c0dcff05ea8..58fa0a33cc5 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -118,6 +118,8 @@ def run_old(self, root) -> List[Module]: def visit(self, node): """Switch execution based on node type""" # print(f"\nvisit {node.type}") + if node == None: + return None if node.type in ["program", "module", "source_file"] : return self.visit_module(node) @@ -153,12 +155,12 @@ def visit(self, node): return self.visit_do_loop_statement(node) elif node.type == "switch_statement": return self.visit_switch_statement(node) + elif node.type == "case_clause": + return self.visit_case_clause(node) + elif node.type == "otherwise_clause": + return self.visit_otherwise_clause(node) elif node.type == "if_statement": return self.visit_if_statement(node) - elif node.type == "elseif_clause": - return self.visit_elseif_clause(node) - elif node.type in ["else_clause", "otherwise_clause"]: - return self.visit_else_clause(node) elif node.type == "derived_type_definition": return self.visit_derived_type(node) elif node.type == "derived_type_member_expression": @@ -597,9 +599,10 @@ def visit_switch_statement(self, node): # get switch identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) - # get n case clauses + # get 0-n case clauses case_clauses = get_children_by_types(node, ["case_clause"]) + # get 0-1 otherwise clauses mi = self.visit(get_first_child_by_type(node, "otherwise_clause")) return mi @@ -607,6 +610,11 @@ def visit_switch_statement(self, node): def visit_if_statement(self, node): """ return a ModelIf if, elseif, and else clauses""" + # ModelIf ( + # expr: Expression + # body: list(body nodes) + # orelse: list(ModelIf) + # if_statement Tree-sitter syntax tree: # if # comparison_operator @@ -615,44 +623,35 @@ def visit_if_statement(self, node): # else_clause (0-1 of these) # end - # the initial ModelIf node is built just like the else-if clause - mi = self.visit_elseif_clause(node) + def model_if_node(_node): + """ return a ModelIf with comparison operator and body nodes. """ + ret = ModelIf() + # expression + ret.expr = self.visit(get_first_child_by_type( + _node, + "comparison_operator" + )) + # body + block = get_first_child_by_type(_node, "block") + if block: + ast_nodes = [self.visit(child) for child in block.children] + ret.body = [ast_node for ast_node in ast_nodes if ast_node] + return ret - # get 0-n elseif_clauses - elseif_clauses = get_children_by_types(node, ["elseif_clause"]) - for child in elseif_clauses: - elseif_node = self.visit(child) - if elseif_node: - if not mi.orelse: - mi.orelse = list() - mi.orelse.append(elseif_node) - - # get 0-1 else_clauses - else_clauses = get_children_by_types(node, ["else_clause"]) - for child in else_clauses: - else_node = self.visit(child) - if else_node.body: - for body_node in else_node.body: - if not mi.orelse: - mi.orelse = list() - mi.orelse.append(body_node) + # the if statement is defined by a ModelIf struct + mi = model_if_node(node) - return mi - - def visit_elseif_clause(self, node): - """ return a ModelIf with comparison operator and body nodes. """ - mi = self.visit_else_clause(node) - comparison_operator = get_first_child_by_type(node, "comparison_operator") - mi.expr = self.visit(comparison_operator) + # get 0-n elseif_clauses as ModelIf structs + elseif_clauses = get_children_by_types(node, ["elseif_clause"]) + mi.orelse = [model_if_node(child) for child in elseif_clauses] - return mi + # if an else clause exists, we add its ModelIf body directly + else_clauses = [get_first_child_by_type(node, "else_clause")] + for body in [model_if_node(e).body for e in else_clauses if e]: + mi.orelse += body - def visit_else_clause(self, node: Node): - """ Return a ModelIf with body nodes from Tree-sitter block nodes. """ - mi = ModelIf() - mi.body = self.get_body_nodes(node) return mi - + def visit_assignment(self, node): left, _, right = node.children @@ -1149,12 +1148,3 @@ def get_gromet_function_node(self, func_name: str) -> Name: return self.variable_context.get_node(func_name) return self.variable_context.add_variable(func_name, "function", None) - - def get_body_nodes(self, node): - """ Return valid body nodes from Tree-sitter block node. """ - block = get_first_child_by_type(node, "block") - ast_nodes = [self.visit(child) for child in block.children] - body_nodes = [ast_node for ast_node in ast_nodes if ast_node] - if len(body_nodes) > 0: - return body_nodes - return None From 40bb9f23c25837d73617f7b43202738d287ac708 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 19:22:58 -0700 Subject: [PATCH 21/84] Cleaned up comments --- .../CAST/matlab/matlab_to_cast.py | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 58fa0a33cc5..4bcbfae1b9f 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -608,13 +608,7 @@ def visit_switch_statement(self, node): return mi def visit_if_statement(self, node): - """ return a ModelIf if, elseif, and else clauses""" - - # ModelIf ( - # expr: Expression - # body: list(body nodes) - # orelse: list(ModelIf) - + """ return a node describing if, elseif, else conditional logic""" # if_statement Tree-sitter syntax tree: # if # comparison_operator @@ -623,34 +617,34 @@ def visit_if_statement(self, node): # else_clause (0-1 of these) # end - def model_if_node(_node): - """ return a ModelIf with comparison operator and body nodes. """ + def conditional(_node): + """ return a ModelIf struct for the conditional logic clause. """ ret = ModelIf() - # expression + # expression, possibly None ret.expr = self.visit(get_first_child_by_type( _node, "comparison_operator" )) - # body + # body, possibly None block = get_first_child_by_type(_node, "block") if block: ast_nodes = [self.visit(child) for child in block.children] ret.body = [ast_node for ast_node in ast_nodes if ast_node] return ret - # the if statement is defined by a ModelIf struct - mi = model_if_node(node) + # the if statement is returned as a ModelIf struct + ret = conditional(node) - # get 0-n elseif_clauses as ModelIf structs + # add 0-n elseif_clauses to the returned or-else list elseif_clauses = get_children_by_types(node, ["elseif_clause"]) - mi.orelse = [model_if_node(child) for child in elseif_clauses] + ret.orelse = [conditional(child) for child in elseif_clauses] - # if an else clause exists, we add its ModelIf body directly + # if an else clause exists, add its block nodes to the returned or-else list else_clauses = [get_first_child_by_type(node, "else_clause")] - for body in [model_if_node(e).body for e in else_clauses if e]: - mi.orelse += body + for body in [conditional(e).body for e in else_clauses]: + ret.orelse += body - return mi + return ret def visit_assignment(self, node): left, _, right = node.children From 8cf81fa0e118a68293c8238cb6c6522b90acfec2 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 19:39:05 -0700 Subject: [PATCH 22/84] Fixed a list handling bug --- .../CAST/matlab/matlab_to_cast.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 4bcbfae1b9f..563d39aa4b1 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -612,20 +612,20 @@ def visit_if_statement(self, node): # if_statement Tree-sitter syntax tree: # if # comparison_operator - # body block with 1-n elements - # elseif_clause (0-n of these) - # else_clause (0-1 of these) + # block with 1-n instructions + # elseif_clause (0-n, with comparison_operator and instruction block) + # else_clause (0-1, instruction block only) # end def conditional(_node): """ return a ModelIf struct for the conditional logic clause. """ ret = ModelIf() - # expression, possibly None + # comparison_operator, possibly None ret.expr = self.visit(get_first_child_by_type( _node, "comparison_operator" )) - # body, possibly None + # instruction_block possibly None block = get_first_child_by_type(_node, "block") if block: ast_nodes = [self.visit(child) for child in block.children] @@ -635,12 +635,12 @@ def conditional(_node): # the if statement is returned as a ModelIf struct ret = conditional(node) - # add 0-n elseif_clauses to the returned or-else list + # add 0-n elseif_clauses to the returned ModelIf or-else list elseif_clauses = get_children_by_types(node, ["elseif_clause"]) ret.orelse = [conditional(child) for child in elseif_clauses] - # if an else clause exists, add its block nodes to the returned or-else list - else_clauses = [get_first_child_by_type(node, "else_clause")] + # 0-1 else clause, add instruction block to the ModelIf or-else list + else_clauses = get_children_by_types(node, ["else_clause"]) for body in [conditional(e).body for e in else_clauses]: ret.orelse += body From 4c1c5490b6ef0f2c5bb301a06975c5c05fe95923 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 20:11:22 -0700 Subject: [PATCH 23/84] Added a test case ifor if-elseif conditional --- .../CAST/matlab/tests/test_conditional.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py index 63183c52824..694dad076be 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_conditional.py +++ b/skema/program_analysis/CAST/matlab/tests/test_conditional.py @@ -44,6 +44,27 @@ def test_if_else(): assert_assignment(mi.orelse[0], left="y", right = "x") assert_assignment(mi.orelse[1], left="foo", right = "'bar'") +def test_if_elseif(): + """ Test CAST from MATLAB 'if elseif else' conditional logic.""" + + source = """ + if x > 5 + y = 6 + elseif x > 0 + y = x + end + """ + + mi = cast_nodes(source)[0] + # if + assert isinstance(mi, ModelIf) + assert_expression(mi.expr, op = ">", left = "x", right = "5") + assert_assignment(mi.body[0], left="y", right = "6") + # elseif + assert isinstance(mi.orelse[0], ModelIf) + assert_expression(mi.orelse[0].expr, op = ">", left = "x", right = "0") + assert_assignment(mi.orelse[0].body[0], left="y", right = "x") + def test_if_elseif_else(): """ Test CAST from MATLAB 'if elseif else' conditional logic.""" From 810d9bb56e4552c2699ab900baf38f7b14b30ded Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 20:11:44 -0700 Subject: [PATCH 24/84] Cleaned up comments --- .../program_analysis/CAST/matlab/matlab_to_cast.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 563d39aa4b1..79fd86d0628 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -609,13 +609,6 @@ def visit_switch_statement(self, node): def visit_if_statement(self, node): """ return a node describing if, elseif, else conditional logic""" - # if_statement Tree-sitter syntax tree: - # if - # comparison_operator - # block with 1-n instructions - # elseif_clause (0-n, with comparison_operator and instruction block) - # else_clause (0-1, instruction block only) - # end def conditional(_node): """ return a ModelIf struct for the conditional logic clause. """ @@ -630,16 +623,15 @@ def conditional(_node): if block: ast_nodes = [self.visit(child) for child in block.children] ret.body = [ast_node for ast_node in ast_nodes if ast_node] + return ret # the if statement is returned as a ModelIf struct ret = conditional(node) - - # add 0-n elseif_clauses to the returned ModelIf or-else list + # add 0-n elseif_clauses elseif_clauses = get_children_by_types(node, ["elseif_clause"]) ret.orelse = [conditional(child) for child in elseif_clauses] - - # 0-1 else clause, add instruction block to the ModelIf or-else list + # add 0-1 else clause else_clauses = get_children_by_types(node, ["else_clause"]) for body in [conditional(e).body for e in else_clauses]: ret.orelse += body From d3742171e53e5ccff4e54295207f0ac23ac23d32 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 20:29:23 -0700 Subject: [PATCH 25/84] Added a guard for None values --- skema/program_analysis/CAST/matlab/matlab_to_cast.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 79fd86d0628..f2b727e311c 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -633,8 +633,9 @@ def conditional(_node): ret.orelse = [conditional(child) for child in elseif_clauses] # add 0-1 else clause else_clauses = get_children_by_types(node, ["else_clause"]) - for body in [conditional(e).body for e in else_clauses]: - ret.orelse += body + for block in [conditional(e).body for e in else_clauses]: + if block: + ret.orelse += block return ret From bc8a32df4256dee021168e42805af318058dff4b Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 02:40:59 -0700 Subject: [PATCH 26/84] switch statement literal nodes --- .../CAST/matlab/matlab_to_cast.py | 94 +++++++++++++------ 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index f2b727e311c..aa0df79a5fc 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -155,10 +155,6 @@ def visit(self, node): return self.visit_do_loop_statement(node) elif node.type == "switch_statement": return self.visit_switch_statement(node) - elif node.type == "case_clause": - return self.visit_case_clause(node) - elif node.type == "otherwise_clause": - return self.visit_otherwise_clause(node) elif node.type == "if_statement": return self.visit_if_statement(node) elif node.type == "derived_type_definition": @@ -576,64 +572,100 @@ def visit_otherwise_clause(self, node): return mi def visit_switch_statement(self, node): - """ return a conditional logic statement based on the switch statement """ - # case 'one' - # n = 1; - # case {2, '2', 'two'} - # n = 2 - # otherwise - # n = 0; - # end - # - # The above statement can be reduced to: - # if s in ['one'] - # n = 1 - # elseif s in [2, '2', 'two'] - # n = 2 - # else - # n = 0 - # The first case clause becomes an if statement - # any subsequent case clauses become elseif statements - # a default clause, if one exists, becomes an else statement + """ return a conditional statement based on the switch statement """ + + literal_types = ["number", "string", "boolean", "array_literal"] + sequence_types = ["cell", "row"] + + def get_literals(literal_node, target_list): + # keep looking + sequences = get_children_by_types(literal_node, sequence_types) + for sequence in sequences: + get_literals(sequence, target_list) + # found + literals = get_children_by_types(literal_node, literal_types) + print(f"get_literals: Literals found: {len(literals)}") + ast_nodes = [self.visit(child) for child in literals] + target_list += [ast_node for ast_node in ast_nodes if ast_node] + return target_list + + + return Operator( + source_language="matlab", + interpreter=None, + version=None, + op=op, + operands=operands, + source_refs=[self.node_helper.get_source_ref(node)], + ) + def conditional(conditional_node, identifier): + """ Create a sequence of if-then conditionals """ + ret=ModelIf() + + # find target values + literals = get_literals(conditional_node, list()) + print(f"conditional: Literals found: {len(literals)}") + ret.orelse = literals + + # comparison operator + ret.expr = self.visit(get_first_child_by_type( + conditional_node, + "comparison_operator" + )) + + # instruction_block possibly None + block = get_first_child_by_type(conditional_node, "block") + if block: + ast_nodes = [self.visit(child) for child in block.children] + ret.body = [ast_node for ast_node in ast_nodes if ast_node] + + return ret # get switch identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) + # Create a placeholder ModelIf for now + ret = ModelIf() + # get 0-n case clauses case_clauses = get_children_by_types(node, ["case_clause"]) + ret.orelse = [conditional(child, identifier) for child in case_clauses] # get 0-1 otherwise clauses - mi = self.visit(get_first_child_by_type(node, "otherwise_clause")) + otherwise_clauses = get_children_by_types(node, ["otherwise_clause"]) + for block in [conditional(child, identifier).body for child in otherwise_clauses]: + if block: + ret.orelse += block - return mi + return ret def visit_if_statement(self, node): """ return a node describing if, elseif, else conditional logic""" - def conditional(_node): - """ return a ModelIf struct for the conditional logic clause. """ + def conditional(conditional_node): + """ return a ModelIf struct for the conditional logic node. """ ret = ModelIf() # comparison_operator, possibly None ret.expr = self.visit(get_first_child_by_type( - _node, + conditional_node, "comparison_operator" )) # instruction_block possibly None - block = get_first_child_by_type(_node, "block") + block = get_first_child_by_type(conditional_node, "block") if block: ast_nodes = [self.visit(child) for child in block.children] ret.body = [ast_node for ast_node in ast_nodes if ast_node] return ret - # the if statement is returned as a ModelIf struct + # the if statement is returned as a ModelIf AstNode ret = conditional(node) # add 0-n elseif_clauses elseif_clauses = get_children_by_types(node, ["elseif_clause"]) ret.orelse = [conditional(child) for child in elseif_clauses] # add 0-1 else clause else_clauses = get_children_by_types(node, ["else_clause"]) - for block in [conditional(e).body for e in else_clauses]: + for block in [conditional(child).body for child in else_clauses]: if block: ret.orelse += block From bd0be4d1341a51730c0d953256fe71b22f772ca6 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 07:17:52 -0700 Subject: [PATCH 27/84] fixed conditional orelse logic, added switch statement --- .../CAST/matlab/matlab_to_cast.py | 107 ++++++++++-------- 1 file changed, 58 insertions(+), 49 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index aa0df79a5fc..caebce2206c 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -577,67 +577,67 @@ def visit_switch_statement(self, node): literal_types = ["number", "string", "boolean", "array_literal"] sequence_types = ["cell", "row"] - def get_literals(literal_node, target_list): - # keep looking - sequences = get_children_by_types(literal_node, sequence_types) - for sequence in sequences: - get_literals(sequence, target_list) - # found - literals = get_children_by_types(literal_node, literal_types) - print(f"get_literals: Literals found: {len(literals)}") - ast_nodes = [self.visit(child) for child in literals] - target_list += [ast_node for ast_node in ast_nodes if ast_node] - return target_list - + def get_literals(node, literals): + for child in node.children: + get_literals(child, literals + get_children_by_types(child, literal_types)) + return [self.visit(literal) for literal in literals] + + def get_operator(op, left, right): + """ Return a comparison operator between identifier and literal""" + return Operator( + source_language="matlab", + interpreter=None, + version=None, + op=op, + operands=[left, right] + # source_refs=[self.node_helper.get_source_ref(right)] + ) - return Operator( - source_language="matlab", - interpreter=None, - version=None, - op=op, - operands=operands, - source_refs=[self.node_helper.get_source_ref(node)], - ) - def conditional(conditional_node, identifier): + def conditional(conditional_node: Node, identifier): """ Create a sequence of if-then conditionals """ - ret=ModelIf() - - # find target values - literals = get_literals(conditional_node, list()) - print(f"conditional: Literals found: {len(literals)}") - ret.orelse = literals + mi=ModelIf() + + # there will either be a list containing a single literal_value + literal_values = get_children_by_types(conditional_node, literal_types) + ast_nodes = [self.visit(element) for element in literal_values] + valid_nodes = [ast_node for ast_node in ast_nodes if ast_node] + if len(valid_nodes) == 1: + mi.expr = get_operator("==", identifier, valid_nodes[0]) + + # else there will be a nested sequence containing them. + multiple_values = get_children_by_types(conditional_node, sequence_types) + # ... chain ifelse - # comparison operator - ret.expr = self.visit(get_first_child_by_type( - conditional_node, - "comparison_operator" - )) - # instruction_block possibly None + # instruction_block posibly None block = get_first_child_by_type(conditional_node, "block") if block: ast_nodes = [self.visit(child) for child in block.children] - ret.body = [ast_node for ast_node in ast_nodes if ast_node] + mi.body = [ast_node for ast_node in ast_nodes if ast_node] - return ret + return mi # get switch identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) - # Create a placeholder ModelIf for now - ret = ModelIf() - # get 0-n case clauses case_clauses = get_children_by_types(node, ["case_clause"]) - ret.orelse = [conditional(child, identifier) for child in case_clauses] + model_ifs = [conditional(child, identifier) for child in case_clauses] + + # link model_ifs as orelse lists + for i, model_if in enumerate(model_ifs[1:]): + model_ifs[i].orelse = [model_if] # get 0-1 otherwise clauses - otherwise_clauses = get_children_by_types(node, ["otherwise_clause"]) - for block in [conditional(child, identifier).body for child in otherwise_clauses]: + otherwise_clause = get_first_child_by_type(node, "otherwise_clause") + if otherwise_clause: + block = get_first_child_by_type(otherwise_clause, "block") if block: - ret.orelse += block + ast_nodes = [self.visit(child) for child in block.children] + last = model_ifs[len(model_ifs)-1] + last.orelse = [ast_node for ast_node in ast_nodes if ast_node] - return ret + return model_ifs[0] def visit_if_statement(self, node): """ return a node describing if, elseif, else conditional logic""" @@ -659,17 +659,26 @@ def conditional(conditional_node): return ret # the if statement is returned as a ModelIf AstNode - ret = conditional(node) + model_ifs = [conditional(node)] # add 0-n elseif_clauses elseif_clauses = get_children_by_types(node, ["elseif_clause"]) - ret.orelse = [conditional(child) for child in elseif_clauses] + model_ifs += [conditional(child) for child in elseif_clauses] + + # link model_ifs as orelse lists + for i, model_if in enumerate(model_ifs[1:]): + model_ifs[i].orelse = [model_if] + # add 0-1 else clause - else_clauses = get_children_by_types(node, ["else_clause"]) - for block in [conditional(child).body for child in else_clauses]: + else_clauses = [get_first_child_by_type(node, "else_clause")] + for else_clause in else_clauses: + block = get_first_child_by_type(else_clause, "block") if block: - ret.orelse += block + ast_nodes = [self.visit(child) for child in block.children] + last = model_ifs[len(model_ifs)-1] + last.orelse = [ast_node for ast_node in ast_nodes if ast_node] + + return model_ifs[0] - return ret def visit_assignment(self, node): left, _, right = node.children From 0d2f2bfbede4c1cc99a840279fea1eb94a034c9c Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 07:32:13 -0700 Subject: [PATCH 28/84] else clause node bugfix --- skema/program_analysis/CAST/matlab/matlab_to_cast.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index caebce2206c..81fc72582f5 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -669,8 +669,8 @@ def conditional(conditional_node): model_ifs[i].orelse = [model_if] # add 0-1 else clause - else_clauses = [get_first_child_by_type(node, "else_clause")] - for else_clause in else_clauses: + else_clause = get_first_child_by_type(node, "else_clause") + if else_clause: block = get_first_child_by_type(else_clause, "block") if block: ast_nodes = [self.visit(child) for child in block.children] From a83ac8a9addcbc6c5bf3929c644f0fe77e096e3e Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 07:38:17 -0700 Subject: [PATCH 29/84] Fixed a bug in a test case --- skema/program_analysis/CAST/matlab/tests/test_conditional.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py index 694dad076be..6c8601f625d 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_conditional.py +++ b/skema/program_analysis/CAST/matlab/tests/test_conditional.py @@ -88,4 +88,4 @@ def test_if_elseif_else(): assert_expression(mi.orelse[0].expr, op = ">", left = "x", right = "0") assert_assignment(mi.orelse[0].body[0], left="y", right = "x") # else - assert_assignment(mi.orelse[1], left="y", right = "0") + assert_assignment(mi.orelse[0].orelse[0], left="y", right = "0") From 68b66d8a1df76b039225f66445eeb2d71a593d8f Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 07:54:08 -0700 Subject: [PATCH 30/84] Removed an outdated method, fixed comments --- .../CAST/matlab/matlab_to_cast.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 81fc72582f5..c0f93e4a15c 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -577,11 +577,6 @@ def visit_switch_statement(self, node): literal_types = ["number", "string", "boolean", "array_literal"] sequence_types = ["cell", "row"] - def get_literals(node, literals): - for child in node.children: - get_literals(child, literals + get_children_by_types(child, literal_types)) - return [self.visit(literal) for literal in literals] - def get_operator(op, left, right): """ Return a comparison operator between identifier and literal""" return Operator( @@ -597,16 +592,17 @@ def conditional(conditional_node: Node, identifier): """ Create a sequence of if-then conditionals """ mi=ModelIf() - # there will either be a list containing a single literal_value + # there will either be a list containing a single literal_value... literal_values = get_children_by_types(conditional_node, literal_types) ast_nodes = [self.visit(element) for element in literal_values] valid_nodes = [ast_node for ast_node in ast_nodes if ast_node] if len(valid_nodes) == 1: mi.expr = get_operator("==", identifier, valid_nodes[0]) - # else there will be a nested sequence containing them. + # ...or there will be a nested sequence containing them. multiple_values = get_children_by_types(conditional_node, sequence_types) - # ... chain ifelse + + # ... chain ifelse or test if value in list? # instruction_block posibly None @@ -660,15 +656,16 @@ def conditional(conditional_node): # the if statement is returned as a ModelIf AstNode model_ifs = [conditional(node)] + # add 0-n elseif_clauses elseif_clauses = get_children_by_types(node, ["elseif_clause"]) model_ifs += [conditional(child) for child in elseif_clauses] - # link model_ifs as orelse lists + # chain model_ifs as orelse lists for i, model_if in enumerate(model_ifs[1:]): model_ifs[i].orelse = [model_if] - # add 0-1 else clause + # add 0-1 else clause else_clause = get_first_child_by_type(node, "else_clause") if else_clause: block = get_first_child_by_type(else_clause, "block") From ed22d6d02c08595e5f4f11021f6e09113fd1969b Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 08:08:36 -0700 Subject: [PATCH 31/84] Added a test for switch statements --- .../CAST/matlab/tests/test_switch.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_switch.py b/skema/program_analysis/CAST/matlab/tests/test_switch.py index 34b51380226..f0b082c0e41 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_switch.py +++ b/skema/program_analysis/CAST/matlab/tests/test_switch.py @@ -1,11 +1,11 @@ from skema.program_analysis.CAST.matlab.tests.utils import ( - assert_var, + assert_assignment, assert_expression, cast_nodes ) -from skema.program_analysis.CAST2FN.model.cast import Assignment +from skema.program_analysis.CAST2FN.model.cast import ModelIf -def do_not_test_switch(): +def test_switch_single_values(): """ Test CAST from MATLAB switch statement.""" source = """ @@ -14,10 +14,25 @@ def do_not_test_switch(): n = 1; case 'two' n = 2; + x = y otherwise n = 0; end """ - # The root of the CAST should be Assignment - nodes = cast_nodes(source) + # The root of the CAST should a ModelIf instance + mi0 = cast_nodes(source)[0] + + # case 'one' + assert isinstance(mi0, ModelIf) + assert_assignment(mi0.body[0], left="n", right = "1") + assert_expression(mi0.expr, op="==", left = "s", right = "'one'") + + # case 'two' + mi1 = mi0.orelse[0] + assert isinstance(mi1, ModelIf) + assert_assignment(mi1.body[0], left="n", right = "2") + assert_assignment(mi1.body[1], left="x", right = "y") + + # otherwise + assert_assignment(mi1.orelse[0], left="n", right = "0") From 6936daa4640d643b17e9ff5385ec11498135c96d Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 16:13:45 -0700 Subject: [PATCH 32/84] Added helper methods --- skema/program_analysis/CAST/matlab/node_helper.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/skema/program_analysis/CAST/matlab/node_helper.py b/skema/program_analysis/CAST/matlab/node_helper.py index a76a5e91706..3ef9c3fc083 100644 --- a/skema/program_analysis/CAST/matlab/node_helper.py +++ b/skema/program_analysis/CAST/matlab/node_helper.py @@ -88,6 +88,21 @@ def get_first_child_index(node, type: str): if child.type == type: return i +def get_all(node: Node, types: List, found: List = list()): + """ return nodes with type in types in the entire node tree """ + for child in node.children: + if child.type in types: + found.append(child) + get_all( + node = child, + types = types, + found = found + ) + return found + +def valid(nodes): + """ return the node list without any None elements """ + return [node for node in nodes if node] def remove_comments(node: Node): """Remove comment nodes from tree-sitter parse tree""" From 5f16fc36054c7c8ccbc37939785683ca10a10e2a Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 22:11:57 -0700 Subject: [PATCH 33/84] Added a node_helper method to recursively search a CAST node --- .../CAST/matlab/node_helper.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/node_helper.py b/skema/program_analysis/CAST/matlab/node_helper.py index 3ef9c3fc083..bda494b5f2b 100644 --- a/skema/program_analysis/CAST/matlab/node_helper.py +++ b/skema/program_analysis/CAST/matlab/node_helper.py @@ -88,17 +88,15 @@ def get_first_child_index(node, type: str): if child.type == type: return i -def get_all(node: Node, types: List, found: List = list()): - """ return nodes with type in types in the entire node tree """ - for child in node.children: - if child.type in types: - found.append(child) - get_all( - node = child, - types = types, - found = found - ) - return found +def get_all(node, types): + """ return all nodes with type in types from the entire node tree """ + def search(node, types, ret): + if node.type in types: + ret += [node] + for child in node.children: + search(child, types, ret) + return ret + return search(node, types, []) def valid(nodes): """ return the node list without any None elements """ From 1bfe67eb801b564ad2d75f8a769d32a4146f09ea Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 23:15:35 -0700 Subject: [PATCH 34/84] Added multi-value switch statement case clauses --- .../CAST/matlab/matlab_to_cast.py | 110 ++++++++++-------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index c0f93e4a15c..f48f4f42e6c 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -30,6 +30,7 @@ from skema.program_analysis.CAST.matlab.variable_context import VariableContext from skema.program_analysis.CAST.matlab.node_helper import ( + get_all, get_children_by_types, get_control_children, get_first_child_by_type, @@ -38,6 +39,7 @@ get_non_control_children, remove_comments, NodeHelper, + valid ) from skema.program_analysis.tree_sitter_parsers.build_parsers import INSTALLED_LANGUAGES_FILEPATH @@ -45,6 +47,9 @@ MATLAB_VERSION='matlab_version_here' class MatlabToCast(object): + + literal_types = ["number","string", "boolean", "array_literal"] + def __init__(self, source_path = "", source = ""): # if a source file path is provided, read source from file @@ -145,7 +150,7 @@ def visit(self, node): "math_expression", "relational_expression" ]: return self.visit_math_expression(node) - elif node.type in ["number", "array", "string", "boolean"]: + elif node.type in self.literal_types: return self.visit_literal(node) elif node.type == "keyword_statement": return self.visit_keyword_statement(node) @@ -564,74 +569,80 @@ def visit_do_loop_statement(self, node) -> Loop: source_refs=[self.node_helper.get_source_ref(node)], ) - def visit_otherwise_clause(self, node): - mi = ModelIf() - - # ... - - return mi def visit_switch_statement(self, node): """ return a conditional statement based on the switch statement """ + + # node types used for case comparison + case_node_types = self.literal_types + ["identifier"] - literal_types = ["number", "string", "boolean", "array_literal"] - sequence_types = ["cell", "row"] - + def get_value(ast_node): + """ return the value or var name of the CAST node """ + if isinstance(ast_node, Var): + return ast_node.val.name + return ast_node.value + + def multiple_case_values(cell_node): + """" return a literalValue List with literal values and var names """ + nodes = get_all(cell_node, case_node_types) + ast_nodes = valid([self.visit(node) for node in nodes]) + return LiteralValue( + value_type="List", + value = [get_value(ast_node) for ast_node in ast_nodes], + source_code_data_type=["matlab", MATLAB_VERSION, "unknown"] +# source_refs=[self.node_helper.get_source_ref(cell_node)] + ) + def get_operator(op, left, right): """ Return a comparison operator between identifier and literal""" return Operator( source_language="matlab", interpreter=None, - version=None, + version=MATLAB_VERSION, op=op, operands=[left, right] - # source_refs=[self.node_helper.get_source_ref(right)] ) - def conditional(conditional_node: Node, identifier): - """ Create a sequence of if-then conditionals """ - mi=ModelIf() - - # there will either be a list containing a single literal_value... - literal_values = get_children_by_types(conditional_node, literal_types) - ast_nodes = [self.visit(element) for element in literal_values] - valid_nodes = [ast_node for ast_node in ast_nodes if ast_node] - if len(valid_nodes) == 1: - mi.expr = get_operator("==", identifier, valid_nodes[0]) - - # ...or there will be a nested sequence containing them. - multiple_values = get_children_by_types(conditional_node, sequence_types) - - # ... chain ifelse or test if value in list? - - - # instruction_block posibly None - block = get_first_child_by_type(conditional_node, "block") + def get_expr(case_node, identifier): + """ return an operator for the case_node """ + # multiple case values + cell_node = get_first_child_by_type(case_node, "cell") + if (cell_node): + return get_operator("in", identifier, multiple_case_values(cell_node)) + # single case value + nodes = get_children_by_types(case_node, case_node_types) + ast_nodes = valid([self.visit(element) for element in nodes]) + return get_operator("==", identifier, ast_nodes[0]) + + def get_block(case_node): + """ return the instruction block for the case_node """ + block = get_first_child_by_type(case_node, "block") if block: - ast_nodes = [self.visit(child) for child in block.children] - mi.body = [ast_node for ast_node in ast_nodes if ast_node] + return valid([self.visit(child) for child in block.children]) + return None - return mi - - # get switch identifier + def model_if(case_node, identifier): + return ModelIf( + expr = get_expr(case_node, identifier), + body = get_block(case_node), + ) + + # switch statement identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) - - # get 0-n case clauses - case_clauses = get_children_by_types(node, ["case_clause"]) - model_ifs = [conditional(child, identifier) for child in case_clauses] - - # link model_ifs as orelse lists + + # 1-n case clauses linked with orelse logic + case_nodes = get_children_by_types(node, ["case_clause"]) + model_ifs=[model_if(case_node, identifier) for case_node in case_nodes] for i, model_if in enumerate(model_ifs[1:]): model_ifs[i].orelse = [model_if] - # get 0-1 otherwise clauses + # 0-1 otherwise clauses linked with else logic otherwise_clause = get_first_child_by_type(node, "otherwise_clause") if otherwise_clause: block = get_first_child_by_type(otherwise_clause, "block") if block: - ast_nodes = [self.visit(child) for child in block.children] last = model_ifs[len(model_ifs)-1] - last.orelse = [ast_node for ast_node in ast_nodes if ast_node] + last.orelse = valid([self.visit(child) for child in block.children]) return model_ifs[0] @@ -649,8 +660,7 @@ def conditional(conditional_node): # instruction_block possibly None block = get_first_child_by_type(conditional_node, "block") if block: - ast_nodes = [self.visit(child) for child in block.children] - ret.body = [ast_node for ast_node in ast_nodes if ast_node] + ret.body = valid([self.visit(child) for child in block.children]) return ret @@ -661,7 +671,7 @@ def conditional(conditional_node): elseif_clauses = get_children_by_types(node, ["elseif_clause"]) model_ifs += [conditional(child) for child in elseif_clauses] - # chain model_ifs as orelse lists + # chain model_ifs as orelse logic for i, model_if in enumerate(model_ifs[1:]): model_ifs[i].orelse = [model_if] @@ -670,12 +680,10 @@ def conditional(conditional_node): if else_clause: block = get_first_child_by_type(else_clause, "block") if block: - ast_nodes = [self.visit(child) for child in block.children] last = model_ifs[len(model_ifs)-1] - last.orelse = [ast_node for ast_node in ast_nodes if ast_node] + last.orelse = valid([self.visit(child) for child in block.children]) return model_ifs[0] - def visit_assignment(self, node): left, _, right = node.children From 590bf955054a367ef81655b39a659f6aa64b717b Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 23:17:00 -0700 Subject: [PATCH 35/84] Added CI test case for switch case clauses with multiple values --- .../CAST/matlab/tests/test_switch.py | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_switch.py b/skema/program_analysis/CAST/matlab/tests/test_switch.py index f0b082c0e41..8561fd2076a 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_switch.py +++ b/skema/program_analysis/CAST/matlab/tests/test_switch.py @@ -14,25 +14,57 @@ def test_switch_single_values(): n = 1; case 'two' n = 2; - x = y + x = y; otherwise n = 0; end """ - # The root of the CAST should a ModelIf instance + # case clause 'one' mi0 = cast_nodes(source)[0] - - # case 'one' assert isinstance(mi0, ModelIf) assert_assignment(mi0.body[0], left="n", right = "1") assert_expression(mi0.expr, op="==", left = "s", right = "'one'") - # case 'two' + # case clause 'two' mi1 = mi0.orelse[0] assert isinstance(mi1, ModelIf) assert_assignment(mi1.body[0], left="n", right = "2") assert_assignment(mi1.body[1], left="x", right = "y") - # otherwise + # otherwise clause + assert_assignment(mi1.orelse[0], left="n", right = "0") + +def test_switch_multiple_values(): + """ Test CAST from MATLAB switch statement.""" + + source = """ + switch s + case {'one', 'two', 'three'} + n = 1; + case 2 + n = 2; + otherwise + n = 0; + end + """ + + # case clause {'one', 'two', 'three'} + mi0 = cast_nodes(source)[0] + assert isinstance(mi0, ModelIf) + assert_assignment(mi0.body[0], left="n", right = "1") + assert_expression( + mi0.expr, + op="in", + left = 's', + right = ["'one'", "'two'", "'three'"] + ) + + # case clause 2 + mi1 = mi0.orelse[0] + assert isinstance(mi1, ModelIf) + assert_assignment(mi1.body[0], left="n", right = "2") + assert_expression(mi1.expr, op="==", left = 's', right = 2) + + # otherwise clause assert_assignment(mi1.orelse[0], left="n", right = "0") From b625c0569dc1855689cd38f4cf0292cd4041359d Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 1 Nov 2023 14:18:35 -0700 Subject: [PATCH 36/84] Fixed a tyypo --- skema/program_analysis/CAST/matlab/tests/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/utils.py b/skema/program_analysis/CAST/matlab/tests/utils.py index 73a7cf49300..89f6e405624 100644 --- a/skema/program_analysis/CAST/matlab/tests/utils.py +++ b/skema/program_analysis/CAST/matlab/tests/utils.py @@ -29,7 +29,7 @@ def assert_operand(operand, value = ""): assert(False) def assert_assignment(assignment, left = "", right = ""): - """ Test an Assignment correct type and operands. """ + """ Test an Assignment for correct type and operands. """ assert isinstance(assignment, Assignment) assert_operand(assignment.left, left) assert_operand(assignment.right, right) @@ -43,16 +43,13 @@ def assert_expression(expression, op = "", left = "", right = ""): def first_cast_node(source): """ Return the first node from the first Module of MatlabToCast output """ - # there should only be one CAST object in the cast output list cast = MatlabToCast(source = source).out_cast assert len(cast) == 1 - # there should be one module in the CAST object assert len(cast[0].nodes) == 1 module = cast[0].nodes[0] assert isinstance(module, Module) - # currently we support one node per module. This may change assert len(module.body) == 1 return module.body[0] From 7c3784da5297152579d02466a5792067a5875543 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 1 Nov 2023 18:32:09 -0700 Subject: [PATCH 37/84] Renamed file --- .../tests/{test_conditional_logic.py => test_conditional.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename skema/program_analysis/CAST/matlab/tests/{test_conditional_logic.py => test_conditional.py} (100%) diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional_logic.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py similarity index 100% rename from skema/program_analysis/CAST/matlab/tests/test_conditional_logic.py rename to skema/program_analysis/CAST/matlab/tests/test_conditional.py From bbfc5c44775fa1932565ebcaf919f37cb5f78dae Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Thu, 2 Nov 2023 01:16:10 -0700 Subject: [PATCH 38/84] Tests can now use multiple CAST objects --- .../CAST/matlab/tests/test_assignment.py | 17 ++++++++------ .../matlab/tests/test_binary_operation.py | 19 ++++++++------- .../CAST/matlab/tests/test_conditional.py | 10 ++++---- .../CAST/matlab/tests/test_switch.py | 23 +++++++++++++++++++ .../CAST/matlab/tests/utils.py | 9 ++++---- 5 files changed, 51 insertions(+), 27 deletions(-) create mode 100644 skema/program_analysis/CAST/matlab/tests/test_switch.py diff --git a/skema/program_analysis/CAST/matlab/tests/test_assignment.py b/skema/program_analysis/CAST/matlab/tests/test_assignment.py index 898432720c4..489e549d1c6 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_assignment.py +++ b/skema/program_analysis/CAST/matlab/tests/test_assignment.py @@ -1,6 +1,6 @@ from skema.program_analysis.CAST.matlab.tests.utils import ( assert_assignment, - first_cast_node + cast_nodes ) # Test the CAST returned by processing the simplest MATLAB assignment @@ -8,10 +8,13 @@ def test_assignment(): """ Test CAST from MATLAB 'assignment' statement.""" - source = 'x = 5' + source = """ + x = 5 + y = 4 + """ - # The root of the CAST should be Assignment - assignment = first_cast_node(source) - - # The module body should contain a single assignment node - assert_assignment(assignment, left = "x", right = "5") + # nodes should be two assignments + nodes = cast_nodes(source) + assert len(nodes) == 2 + assert_assignment(nodes[0], left = "x", right = "5") + assert_assignment(nodes[1], left = "y", right = "4") diff --git a/skema/program_analysis/CAST/matlab/tests/test_binary_operation.py b/skema/program_analysis/CAST/matlab/tests/test_binary_operation.py index b0452445992..2496bdae13c 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_binary_operation.py +++ b/skema/program_analysis/CAST/matlab/tests/test_binary_operation.py @@ -1,23 +1,22 @@ from skema.program_analysis.CAST.matlab.tests.utils import ( assert_var, assert_expression, - first_cast_node + cast_nodes ) from skema.program_analysis.CAST2FN.model.cast import Assignment -# Test the CAST returned by processing the simplest MATLAB binary operation - def test_binary_operation(): """ Test CAST from MATLAB binary operation statement.""" source = 'z = x + y' - # The root of the CAST should be Assignment - assignment = first_cast_node(source) - assert isinstance(assignment, Assignment) + # cast nodes should be one assignment + nodes = cast_nodes(source) + assert len(nodes) == 1 + assert isinstance(nodes[0], Assignment) - # Left operand of this assignment node is the variable - assert_var(assignment.left, name = "z") + # Left assignment operand is the variable + assert_var(nodes[0].left, name = "z") - # right operand of this assignment node is a binary expression - assert_expression(assignment.right, op = "+", left = "x", right = "y") + # right assignment operand is a binary expression + assert_expression(nodes[0].right, op = "+", left = "x", right = "y") diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py index 59f4d18f815..dd73a25676f 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_conditional.py +++ b/skema/program_analysis/CAST/matlab/tests/test_conditional.py @@ -1,7 +1,7 @@ from skema.program_analysis.CAST.matlab.tests.utils import ( assert_assignment, assert_expression, - first_cast_node + cast_nodes ) from skema.program_analysis.CAST2FN.model.cast import ModelIf @@ -14,7 +14,8 @@ def test_if(): end """ - mi = first_cast_node(source) + mi = cast_nodes(source)[0] + # if assert isinstance(mi, ModelIf) assert_expression(mi.expr, op = ">", left = "x", right = "5") @@ -31,7 +32,7 @@ def test_if_else(): end """ - mi = first_cast_node(source) + mi = cast_nodes(source)[0] # if assert isinstance(mi, ModelIf) assert_expression(mi.expr, op = ">", left = "x", right = "5") @@ -52,7 +53,7 @@ def test_if_elseif_else(): end """ - mi = first_cast_node(source) + mi = cast_nodes(source)[0] # if assert isinstance(mi, ModelIf) assert_expression(mi.expr, op = ">", left = "x", right = "5") @@ -63,4 +64,3 @@ def test_if_elseif_else(): assert_assignment(mi.orelse[0].body[0], left="y", right = "x") # else assert_assignment(mi.orelse[1], left="y", right = "0") - diff --git a/skema/program_analysis/CAST/matlab/tests/test_switch.py b/skema/program_analysis/CAST/matlab/tests/test_switch.py new file mode 100644 index 00000000000..b1fdb12dda2 --- /dev/null +++ b/skema/program_analysis/CAST/matlab/tests/test_switch.py @@ -0,0 +1,23 @@ +from skema.program_analysis.CAST.matlab.tests.utils import ( + assert_var, + assert_expression, + cast_nodes +) +from skema.program_analysis.CAST2FN.model.cast import Assignment + +def do_not_test_switch(): + """ Test CAST from MATLAB switch statement.""" + + source = """ + switch s + case 'axis' + n = 0; + case 'bottom' + n = 1; + case 'top' + n = 2; + end + """ + + # The root of the CAST should be Assignment + nodes = cast_nodes(source) diff --git a/skema/program_analysis/CAST/matlab/tests/utils.py b/skema/program_analysis/CAST/matlab/tests/utils.py index 89f6e405624..68e1e65b0d1 100644 --- a/skema/program_analysis/CAST/matlab/tests/utils.py +++ b/skema/program_analysis/CAST/matlab/tests/utils.py @@ -41,8 +41,8 @@ def assert_expression(expression, op = "", left = "", right = ""): assert_operand(expression.operands[0], left) assert_operand(expression.operands[1], right) -def first_cast_node(source): - """ Return the first node from the first Module of MatlabToCast output """ +def cast_nodes(source): + """ Return the CAST nodes from the first Module of MatlabToCast output """ # there should only be one CAST object in the cast output list cast = MatlabToCast(source = source).out_cast assert len(cast) == 1 @@ -50,6 +50,5 @@ def first_cast_node(source): assert len(cast[0].nodes) == 1 module = cast[0].nodes[0] assert isinstance(module, Module) - # currently we support one node per module. This may change - assert len(module.body) == 1 - return module.body[0] + # return the module body node list + return module.body From a902d868b23c1a55ecaad4317eefbf5fe7ce07cd Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Thu, 2 Nov 2023 01:33:34 -0700 Subject: [PATCH 39/84] Added a string coparison to the assignment tests --- skema/program_analysis/CAST/matlab/tests/test_assignment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_assignment.py b/skema/program_analysis/CAST/matlab/tests/test_assignment.py index 489e549d1c6..08fd89af13a 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_assignment.py +++ b/skema/program_analysis/CAST/matlab/tests/test_assignment.py @@ -10,11 +10,13 @@ def test_assignment(): source = """ x = 5 - y = 4 + y = "xxx" """ # nodes should be two assignments nodes = cast_nodes(source) assert len(nodes) == 2 assert_assignment(nodes[0], left = "x", right = "5") - assert_assignment(nodes[1], left = "y", right = "4") + + # When comparing strings you must include escaped quotes + assert_assignment(nodes[1], left = "y", right = "\"xxx\"") From 2476607a6715cd2ac3fa4066e22510bed42342c4 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Thu, 2 Nov 2023 16:34:30 -0700 Subject: [PATCH 40/84] Cast debug with multiple input files --- skema/program_analysis/CAST/matlab/cast_out | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index 91cb0adcc04..2b27ee6c63d 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -3,24 +3,28 @@ import json import sys from skema.program_analysis.CAST.matlab.matlab_to_cast import MatlabToCast -""" Run a file of any type through the Tree-sitter MATLAB parser""" + +def show_cast(filename): + """ Run a file of any type through the Tree-sitter MATLAB parser""" + parser = MatlabToCast(filename) + print("\n\nINPUT:") + print(parser.filename) + print("\nSOURCE:") + print(parser.source) + print('\nCAST:') + cast_list = parser.out_cast + for cast in cast_list: + jd = json.dumps( + cast.to_json_object(), + sort_keys=True, + indent=2, + ) + print(jd) if __name__ == "__main__": if len(sys.argv) > 1: for i in range(1, len(sys.argv)): - parser = MatlabToCast(sys.argv[i]) - print("\n\nINPUT:") - print(parser.filename) - print("\nSOURCE:") - print(parser.source) - print('\nCAST:') - cast_list = parser.out_cast - for cast_index in range(0, len(cast_list)): - jd = json.dumps( - cast_list[cast_index].to_json_object(), - sort_keys=True, - indent=2, - ) - print(jd) + show_cast(sys.argv[i]) else: print("Please enter one filename to parse") + From 2ffe426cfbf0d82afba6149cb99be3c5fa7301e9 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Fri, 3 Nov 2023 12:03:54 -0700 Subject: [PATCH 41/84] Added a method for pruning keys from cast debug output --- skema/program_analysis/CAST/matlab/cast_out | 30 ++++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index 2b27ee6c63d..806304622f7 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -3,23 +3,39 @@ import json import sys from skema.program_analysis.CAST.matlab.matlab_to_cast import MatlabToCast +def prune(json_obj, target_key): + """ remove all instances of the target key from the json object""" + def remove_key(json_obj, target_key): + if isinstance(json_obj, dict): + if target_key in json_obj.keys(): + json_obj.pop(target_key) + for key in json_obj.keys(): + remove_key(json_obj[key], target_key) + elif isinstance(json_obj, list): + for item in json_obj: + remove_key(item, target_key) + return(json_obj) + + print(f"Removed key: {target_key}") + return remove_key(json_obj, target_key) def show_cast(filename): """ Run a file of any type through the Tree-sitter MATLAB parser""" parser = MatlabToCast(filename) - print("\n\nINPUT:") + print("\nINPUT FILE:") print(parser.filename) print("\nSOURCE:") print(parser.source) print('\nCAST:') cast_list = parser.out_cast + for cast in cast_list: - jd = json.dumps( - cast.to_json_object(), - sort_keys=True, - indent=2, - ) - print(jd) + json_obj = cast.to_json_object() + + # declutter + json_obj = prune(json_obj, "source_refs") + + print(json.dumps(json_obj, sort_keys = True, indent = 2)) if __name__ == "__main__": if len(sys.argv) > 1: From 78746a2b5e9d2df32c57e09ac388cd6472ba0121 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Fri, 3 Nov 2023 14:11:32 -0700 Subject: [PATCH 42/84] early switch statment handler --- .../CAST/matlab/matlab_to_cast.py | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index d29bc8bfeb0..8e13d3e7220 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -78,6 +78,7 @@ def generate_cast(self) -> List[CAST]: """Interface for generating CAST.""" # remove comments from tree before processing + modules = self.run(remove_comments(self.tree.root_node)) return [CAST([module], "matlab") for module in modules] @@ -96,7 +97,6 @@ def run_old(self, root) -> List[Module]: "function_definition", "subroutine", "assignment", - "switch_statement" ] outer_body_nodes = get_children_by_types(root, body_node_names) @@ -152,6 +152,10 @@ def visit(self, node): return self.visit_extent_specifier(node) elif node.type == "do_loop_statement": return self.visit_do_loop_statement(node) + elif node.type == "switch_statement": + return self.visit_switch_statement(node) + elif node.type == "case_clause": + return self.visit_case_clause(node) elif node.type == "if_statement": return self.visit_if_statement(node) elif node.type == "elseif_clause": @@ -578,6 +582,45 @@ def visit_do_loop_statement(self, node) -> Loop: source_refs=[self.node_helper.get_source_ref(node)], ) + + def visit_switch_statement(self, node): + """ return a conditional logic statement based on the case statement """ + # This case statement: + # switch s + # case 'top' + # n = 1; + # case 'bottom' + # n = 0; + # end + # + # can be reduced to: + # if s == ‘top’ + # n = 1 + # elseif s == ‘bottom’ + # n = 2 + # + # The first case clause becomes an if statement + # any subsequent case clauses become elseif statements + # a default clause, if one exists, becomes an else statement + + mi = ModelIf() + + # get switch identifier + identifier = get_first_child_by_type(node, ("identifier")) + + # get n case clauses + types = list() + types.append("case_clause") + case_clauses = get_children_by_types(node, types) + + return mi + + + def visit_case_clause(self, node): + """ return an if statement defining the case clause""" + pass + + def visit_if_statement(self, node): """ return a ModelIf if, elseif, and else clauses""" @@ -621,8 +664,8 @@ def visit_elseif_clause(self, node): # get ModelIf with body nodes mi = self.visit_else_clause(node) # addd comparison operator - comp: Operator = get_first_child_by_type(node, "comparison_operator") - mi.expr = self.visit(comp) + comparison_operator = get_first_child_by_type(node, "comparison_operator") + mi.expr = self.visit(comparison_operator) return mi From 827662127677c4be161bdda7dff2400627749e89 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Fri, 3 Nov 2023 14:35:16 -0700 Subject: [PATCH 43/84] Fixed node_helper args --- skema/program_analysis/CAST/matlab/matlab_to_cast.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 8e13d3e7220..6b35b868930 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -606,12 +606,10 @@ def visit_switch_statement(self, node): mi = ModelIf() # get switch identifier - identifier = get_first_child_by_type(node, ("identifier")) + identifier = get_first_child_by_type(node, "identifier") # get n case clauses - types = list() - types.append("case_clause") - case_clauses = get_children_by_types(node, types) + case_clauses = get_children_by_types(node, ["case_clause"]) return mi @@ -638,7 +636,7 @@ def visit_if_statement(self, node): mi = self.visit_elseif_clause(node) # get 0-n elseif_clauses - elseif_clauses = get_children_by_types(node, ("elseif_clause")) + elseif_clauses = get_children_by_types(node, ["elseif_clause"]) for child in elseif_clauses: elseif_node = self.visit(child) if elseif_node: @@ -647,7 +645,7 @@ def visit_if_statement(self, node): mi.orelse.append(elseif_node) # get 0-1 else_clauses - else_clauses = get_children_by_types(node, ("else_clause")) + else_clauses = get_children_by_types(node, ["else_clause"]) for child in else_clauses: else_node = self.visit(child) if else_node.body: From 0b4d1284e18e6cb51c40c4297c2939926e65273e Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Fri, 3 Nov 2023 15:22:36 -0700 Subject: [PATCH 44/84] cast_out debug with multiple key removal --- skema/program_analysis/CAST/matlab/cast_out | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index 806304622f7..f829467b330 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -1,23 +1,25 @@ #!/usr/bin/env python3 import json +from typing import List import sys from skema.program_analysis.CAST.matlab.matlab_to_cast import MatlabToCast -def prune(json_obj, target_key): - """ remove all instances of the target key from the json object""" +def prune(json_obj, target_keys: List): + """ remove all instances of the target keys from the json object""" def remove_key(json_obj, target_key): if isinstance(json_obj, dict): - if target_key in json_obj.keys(): - json_obj.pop(target_key) - for key in json_obj.keys(): - remove_key(json_obj[key], target_key) + for target_key in target_keys: + if target_key in json_obj.keys(): + json_obj.pop(target_key) + for key in json_obj.keys(): + remove_key(json_obj[key], target_keys) elif isinstance(json_obj, list): for item in json_obj: - remove_key(item, target_key) + remove_key(item, target_keys) return(json_obj) - print(f"Removed key: {target_key}") - return remove_key(json_obj, target_key) + print(f"Removed keys: {target_keys}") + return remove_key(json_obj, target_keys) def show_cast(filename): """ Run a file of any type through the Tree-sitter MATLAB parser""" @@ -33,7 +35,7 @@ def show_cast(filename): json_obj = cast.to_json_object() # declutter - json_obj = prune(json_obj, "source_refs") + json_obj = prune(json_obj, ["source_refs"]) print(json.dumps(json_obj, sort_keys = True, indent = 2)) From 7496d7b29e4457260ada6e813b5d1aea516cf7e4 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Fri, 3 Nov 2023 22:13:31 -0700 Subject: [PATCH 45/84] switch 'if' expression done --- .../CAST/matlab/matlab_to_cast.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 6b35b868930..bb46dee2121 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -154,8 +154,6 @@ def visit(self, node): return self.visit_do_loop_statement(node) elif node.type == "switch_statement": return self.visit_switch_statement(node) - elif node.type == "case_clause": - return self.visit_case_clause(node) elif node.type == "if_statement": return self.visit_if_statement(node) elif node.type == "elseif_clause": @@ -603,20 +601,37 @@ def visit_switch_statement(self, node): # any subsequent case clauses become elseif statements # a default clause, if one exists, becomes an else statement - mi = ModelIf() + def if_clause(identifier, case_clause): + # { + # "body": null, + # "expr": null, + # "node_type": "ModelIf", + # "orelse": null + # } + + operand1 = self.visit(get_first_child_by_type(case_clause, "string")) + + expr = Operator( + op = "==", + operands = [identifier, operand1] + ) + mi = ModelIf(expr = expr) + + return mi + # get switch identifier - identifier = get_first_child_by_type(node, "identifier") + identifier = self.visit(get_first_child_by_type(node, "identifier")) # get n case clauses case_clauses = get_children_by_types(node, ["case_clause"]) - return mi + # use first case clause to create an if statment + if len(case_clauses) > 0: + mi = if_clause(identifier, case_clauses[0]) + return mi - def visit_case_clause(self, node): - """ return an if statement defining the case clause""" - pass def visit_if_statement(self, node): From bf735cd8a35ce7aca12d987a4e4905ad734834c1 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Sat, 4 Nov 2023 20:45:40 -0700 Subject: [PATCH 46/84] Added key filtering and correct reporting of None values --- skema/program_analysis/CAST/matlab/cast_out | 25 +++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index f829467b330..37dada86696 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -1,10 +1,19 @@ #!/usr/bin/env python3 import json -from typing import List import sys from skema.program_analysis.CAST.matlab.matlab_to_cast import MatlabToCast +from typing import List + +# Show a CAST object as JSON with certain keys filtered for clarity. -def prune(json_obj, target_keys: List): +# Keys to remove from the output for clarity. +KEY_FILTER = [ + "source_refs", + "default_value", + "interpreter" +] + +def remove_keys(json_obj, target_keys: List): """ remove all instances of the target keys from the json object""" def remove_key(json_obj, target_key): if isinstance(json_obj, dict): @@ -33,11 +42,13 @@ def show_cast(filename): for cast in cast_list: json_obj = cast.to_json_object() - - # declutter - json_obj = prune(json_obj, ["source_refs"]) - - print(json.dumps(json_obj, sort_keys = True, indent = 2)) + # declutter JSON by filtering keys + json_obj = remove_keys(json_obj, KEY_FILTER) + # pretty print JSON to string + output = json.dumps(json_obj, sort_keys=True, indent=2) + # json.dumps incorrectly reports None as null. Python has no null. + output = output.replace (": null", ": None") + print(output) if __name__ == "__main__": if len(sys.argv) > 1: From 3a8d93d094d7c962831ea2daccfcdf1b66b3287e Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Sat, 4 Nov 2023 20:47:16 -0700 Subject: [PATCH 47/84] Added an 'otherwise' clause to the MATLAB switch statement test --- skema/program_analysis/CAST/matlab/tests/test_switch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_switch.py b/skema/program_analysis/CAST/matlab/tests/test_switch.py index b1fdb12dda2..34b51380226 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_switch.py +++ b/skema/program_analysis/CAST/matlab/tests/test_switch.py @@ -10,12 +10,12 @@ def do_not_test_switch(): source = """ switch s - case 'axis' - n = 0; - case 'bottom' + case 'one' n = 1; - case 'top' + case 'two' n = 2; + otherwise + n = 0; end """ From e024bcb07c82c7547fe15220559dc5a4aac7bfee Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Sat, 4 Nov 2023 20:48:31 -0700 Subject: [PATCH 48/84] Populated Operator fields, fixed comments --- .../CAST/matlab/matlab_to_cast.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index bb46dee2121..6c1df8ef272 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -583,19 +583,23 @@ def visit_do_loop_statement(self, node) -> Loop: def visit_switch_statement(self, node): """ return a conditional logic statement based on the case statement """ - # This case statement: + # This MATLAB case statement: # switch s - # case 'top' + # case 'one' # n = 1; - # case 'bottom' + # case 'two' + # n = 2; + # otherwise # n = 0; # end # # can be reduced to: - # if s == ‘top’ + # if s == 'one' # n = 1 - # elseif s == ‘bottom’ + # elseif s == 'two' # n = 2 + # else + # n = 0 # # The first case clause becomes an if statement # any subsequent case clauses become elseif statements @@ -611,11 +615,13 @@ def if_clause(identifier, case_clause): operand1 = self.visit(get_first_child_by_type(case_clause, "string")) - expr = Operator( + mi = self.visit_else_clause(case_clause) + mi.expr = Operator( op = "==", - operands = [identifier, operand1] + operands = [identifier, operand1], + source_language = "MATLAB", + version = MATLAB_VERSION ) - mi = ModelIf(expr = expr) return mi From 137abcc676978de61e380e6285f00cd9377ecba2 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 11:06:34 -0700 Subject: [PATCH 49/84] Cleaned up code formatting, comments --- skema/program_analysis/CAST/matlab/cast_out | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index 37dada86696..5c196b19073 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -4,18 +4,17 @@ import sys from skema.program_analysis.CAST.matlab.matlab_to_cast import MatlabToCast from typing import List -# Show a CAST object as JSON with certain keys filtered for clarity. +# Show a CAST object as pretty printed JSON with keys filtered for clarity. -# Keys to remove from the output for clarity. -KEY_FILTER = [ - "source_refs", - "default_value", - "interpreter" -] +# Keys to remove from the output +#KEY_FILTER = ["source_refs", "default_value", "interpreter"] +KEY_FILTER = ["source_refs"] def remove_keys(json_obj, target_keys: List): """ remove all instances of the target keys from the json object""" + def remove_key(json_obj, target_key): + """ remove all instances of the target key from the json object""" if isinstance(json_obj, dict): for target_key in target_keys: if target_key in json_obj.keys(): @@ -30,6 +29,7 @@ def remove_keys(json_obj, target_keys: List): print(f"Removed keys: {target_keys}") return remove_key(json_obj, target_keys) + def show_cast(filename): """ Run a file of any type through the Tree-sitter MATLAB parser""" parser = MatlabToCast(filename) @@ -39,21 +39,18 @@ def show_cast(filename): print(parser.source) print('\nCAST:') cast_list = parser.out_cast - for cast in cast_list: json_obj = cast.to_json_object() # declutter JSON by filtering keys json_obj = remove_keys(json_obj, KEY_FILTER) # pretty print JSON to string output = json.dumps(json_obj, sort_keys=True, indent=2) - # json.dumps incorrectly reports None as null. Python has no null. - output = output.replace (": null", ": None") print(output) + if __name__ == "__main__": if len(sys.argv) > 1: for i in range(1, len(sys.argv)): show_cast(sys.argv[i]) else: print("Please enter one filename to parse") - From 477d2315b848f722293493e5e8f83957213a4f15 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 11:07:09 -0700 Subject: [PATCH 50/84] Cleaned up comments --- .../CAST/matlab/matlab_to_cast.py | 64 ++++++------------- 1 file changed, 18 insertions(+), 46 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 6c1df8ef272..943271e6096 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -158,7 +158,7 @@ def visit(self, node): return self.visit_if_statement(node) elif node.type == "elseif_clause": return self.visit_elseif_clause(node) - elif node.type == "else_clause": + elif node.type in ["else_clause", "otherwise_clause"]: return self.visit_else_clause(node) elif node.type == "derived_type_definition": return self.visit_derived_type(node) @@ -580,71 +580,47 @@ def visit_do_loop_statement(self, node) -> Loop: source_refs=[self.node_helper.get_source_ref(node)], ) + def visit_otherwise_clause(self, node): + mi = ModelIf() + + # ... + + return mi def visit_switch_statement(self, node): - """ return a conditional logic statement based on the case statement """ - # This MATLAB case statement: - # switch s + """ return a conditional logic statement based on the switch statement """ # case 'one' # n = 1; - # case 'two' - # n = 2; + # case {2, '2', 'two'} + # n = 2 # otherwise # n = 0; # end # - # can be reduced to: - # if s == 'one' + # The above statement can be reduced to: + # if s in ['one'] # n = 1 - # elseif s == 'two' + # elseif s in [2, '2', 'two'] # n = 2 - # else + # else # n = 0 - # # The first case clause becomes an if statement # any subsequent case clauses become elseif statements # a default clause, if one exists, becomes an else statement - def if_clause(identifier, case_clause): - # { - # "body": null, - # "expr": null, - # "node_type": "ModelIf", - # "orelse": null - # } - - operand1 = self.visit(get_first_child_by_type(case_clause, "string")) - - mi = self.visit_else_clause(case_clause) - mi.expr = Operator( - op = "==", - operands = [identifier, operand1], - source_language = "MATLAB", - version = MATLAB_VERSION - ) - - return mi - - # get switch identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) # get n case clauses case_clauses = get_children_by_types(node, ["case_clause"]) - # use first case clause to create an if statment - if len(case_clauses) > 0: - mi = if_clause(identifier, case_clauses[0]) + mi = self.visit(get_first_child_by_type(node, "otherwise_clause")) return mi - - def visit_if_statement(self, node): """ return a ModelIf if, elseif, and else clauses""" - # print('visit_if_statement') - # if_statement Tree-sitter syntax tree: # if # comparison_operator @@ -677,17 +653,13 @@ def visit_if_statement(self, node): return mi - def visit_elseif_clause(self, node): - """ return a ModelIf with comparison and body nodes. """ - # get ModelIf with body nodes + """ return a ModelIf with comparison operator and body nodes. """ mi = self.visit_else_clause(node) - # addd comparison operator comparison_operator = get_first_child_by_type(node, "comparison_operator") mi.expr = self.visit(comparison_operator) return mi - def visit_else_clause(self, node): """ Return a ModelIf with body nodes only. """ @@ -703,7 +675,6 @@ def visit_else_clause(self, node): return mi - def visit_assignment(self, node): """Docstring""" # print('visit_assignment') @@ -773,7 +744,6 @@ def visit_literal(self, node) -> LiteralValue: source_refs=[literal_source_ref], ) - def visit_identifier(self, node): """Docstring""" # print('visit_identifier') @@ -1223,3 +1193,5 @@ def get_gromet_function_node(self, func_name: str) -> Name: return self.variable_context.get_node(func_name) return self.variable_context.add_variable(func_name, "function", None) + + From bd4906a3e3752db84cfe4ed76ccba21006b5a3ed Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 11:37:04 -0700 Subject: [PATCH 51/84] Removed debug print statements and placeholder Docstrings --- .../CAST/matlab/matlab_to_cast.py | 61 ++++--------------- .../CAST/matlab/node_helper.py | 3 - .../CAST/matlab/variable_context.py | 8 --- 3 files changed, 12 insertions(+), 60 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 943271e6096..c0dcff05ea8 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -46,7 +46,6 @@ class MatlabToCast(object): def __init__(self, source_path = "", source = ""): - """docstring""" # if a source file path is provided, read source from file if not source_path == "": @@ -118,7 +117,7 @@ def run_old(self, root) -> List[Module]: def visit(self, node): """Switch execution based on node type""" - # print(f"\nvisit node type = {node.type}") + # print(f"\nvisit {node.type}") if node.type in ["program", "module", "source_file"] : return self.visit_module(node) @@ -169,7 +168,6 @@ def visit(self, node): def visit_module(self, node: Node) -> Module: """Visitor for program and module statement. Returns a Module object""" - # print('visit_module') self.variable_context.push_context() program_body = [] @@ -190,13 +188,10 @@ def visit_module(self, node: Node) -> Module: def visit_internal_procedures(self, node: Node) -> List[FunctionDef]: """Visitor for internal procedures. Returns list of FunctionDef""" - # print('visit_internal_procedures') internal_procedures = get_children_by_types(node, ["function_definition", "subroutine"]) return [self.visit(procedure) for procedure in internal_procedures] def visit_name(self, node): - """Docstring""" - # print('visit_name') # Node structure # (name) @@ -210,8 +205,6 @@ def visit_name(self, node): ) def visit_function_def(self, node): - """Docstring""" - # print('visit_function_def') # TODO: Refactor function def code to use new helper functions # Node structure # (subroutine) @@ -318,8 +311,6 @@ def visit_function_def(self, node): ) def visit_function_call(self, node): - """Docstring""" - # print('visit_function_call') # Pull relevent nodes if node.type == "subroutine_call": function_node = node.children[1] @@ -361,8 +352,6 @@ def visit_function_call(self, node): ) def visit_keyword_statement(self, node): - """Docstring""" - # print('visit_keyword_statement') # Currently, the only keyword_identifier produced by tree-sitter is Return # However, there may be other instances @@ -398,8 +387,6 @@ def visit_keyword_statement(self, node): ) def visit_use_statement(self, node): - """Docstring""" - # print('visit_use_statement') # (use) # (use) # (module_name) @@ -459,7 +446,6 @@ def visit_do_loop_statement(self, node) -> Loop: (body) ... """ - # print('visit_do_loop_statement') # First check for # TODO: Add do until Loop support while_statement_node = get_first_child_by_type(node, "while_statement") @@ -661,23 +647,13 @@ def visit_elseif_clause(self, node): return mi - def visit_else_clause(self, node): - """ Return a ModelIf with body nodes only. """ - # get the top level body nodes + def visit_else_clause(self, node: Node): + """ Return a ModelIf with body nodes from Tree-sitter block nodes. """ mi = ModelIf() - block = get_first_child_by_type(node, "block") - for child in block.children: - body_node = self.visit(child) - if body_node: - if not mi.body: - mi.body = list() - mi.body.append(body_node) - + mi.body = self.get_body_nodes(node) return mi def visit_assignment(self, node): - """Docstring""" - # print('visit_assignment') left, _, right = node.children return Assignment( @@ -688,7 +664,6 @@ def visit_assignment(self, node): def visit_literal(self, node) -> LiteralValue: """Visitor for literals. Returns a LiteralValue""" - # print('visit_literal') literal_type = node.type literal_value = self.node_helper.get_identifier(node) literal_source_ref = self.node_helper.get_source_ref(node) @@ -745,8 +720,6 @@ def visit_literal(self, node) -> LiteralValue: ) def visit_identifier(self, node): - """Docstring""" - # print('visit_identifier') # By default, this is unknown, but can be updated by other visitors identifier = self.node_helper.get_identifier(node) if self.variable_context.is_variable(identifier): @@ -769,8 +742,6 @@ def visit_identifier(self, node): ) def visit_math_expression(self, node): - """Docstring""" - # print('visit_math_expression') op = self.node_helper.get_identifier( get_control_children(node)[0] ) # The operator will be the first control character @@ -896,8 +867,6 @@ def visit_variable_declaration(self, node) -> List: return vars def visit_extent_specifier(self, node): - """Docstring""" - # print('visit_extent_specifier') # Node structure # (extent_specifier) # (identifier) @@ -938,8 +907,6 @@ def visit_derived_type(self, node: Node) -> RecordDef: (BODY_NODES) ... """ - # print('visit_derived_type') - record_name = self.node_helper.get_identifier( get_first_child_by_type(node, "type_name", recurse=True) @@ -1021,7 +988,6 @@ def visit_derived_type_member_expression(self, node) -> Attribute: (argument_list) (type_member) """ - # print('visit_derived_type_member_expression') # If we are accessing an attribute of a scalar type, we can simply pull the name node from the variable context. # However, if this is a dimensional type, we must convert it to a call to _get. @@ -1048,8 +1014,6 @@ def visit_derived_type_member_expression(self, node) -> Attribute: # NOTE: This function starts with _ because it will never be dispatched to directly. There is not a get node in the tree-sitter parse tree. # From context, we will determine when we are accessing an element of a List, and call this function, def _visit_get(self, node): - """Docstring""" - # print('_visit_get') # Node structure # (call_expression) # (identifier) @@ -1088,8 +1052,6 @@ def _visit_get(self, node): ) def _visit_set(self, node): - """Docstring""" - # print('_visit_set') # Node structure # (assignment) # (call_expression) @@ -1116,7 +1078,6 @@ def _visit_while(self, node) -> Loop: (...) ... (body) ... """ - # print('_visit_while') while_statement_node = get_first_child_by_type(node, "while_statement") # The first body node will be the node after the while_statement @@ -1145,7 +1106,6 @@ def _visit_while(self, node) -> Loop: def _visit_implied_do_loop(self, node) -> Call: """Custom visitor for implied_do_loop array literal. This form gets converted to a call to range""" # TODO: This loop_control is the same as the do loop. Can we turn this into one visitor? - # print('_visit_implied_do_loop') loop_control_node = get_first_child_by_type( node, "loop_control_expression", recurse=True ) @@ -1174,8 +1134,6 @@ def _visit_implied_do_loop(self, node) -> Call: ) def _visit_passthrough(self, node): - """Docstring""" - # print('_visit_passthrough') if len(node.children) == 0: return None @@ -1185,8 +1143,6 @@ def _visit_passthrough(self, node): return child_cast def get_gromet_function_node(self, func_name: str) -> Name: - """Docstring""" - # print('get_gromet_function_node') # Idealy, we would be able to create a dummy node and just call the name visitor. # However, tree-sitter does not allow you to create or modify nodes, so we have to recreate the logic here. if self.variable_context.is_variable(func_name): @@ -1194,4 +1150,11 @@ def get_gromet_function_node(self, func_name: str) -> Name: return self.variable_context.add_variable(func_name, "function", None) - + def get_body_nodes(self, node): + """ Return valid body nodes from Tree-sitter block node. """ + block = get_first_child_by_type(node, "block") + ast_nodes = [self.visit(child) for child in block.children] + body_nodes = [ast_node for ast_node in ast_nodes if ast_node] + if len(body_nodes) > 0: + return body_nodes + return None diff --git a/skema/program_analysis/CAST/matlab/node_helper.py b/skema/program_analysis/CAST/matlab/node_helper.py index a64af619599..a76a5e91706 100644 --- a/skema/program_analysis/CAST/matlab/node_helper.py +++ b/skema/program_analysis/CAST/matlab/node_helper.py @@ -27,7 +27,6 @@ class NodeHelper(): def __init__(self, source: str, source_file_name: str): - """Docstring""" self.source = source self.source_file_name = source_file_name @@ -114,12 +113,10 @@ def get_last_child_index(node, type: str): def get_control_children(node: Node): - """Docstring""" return get_children_by_types(node, CONTROL_CHARACTERS) def get_non_control_children(node: Node): - """Docstring""" children = [] for child in node.children: if child.type not in CONTROL_CHARACTERS: diff --git a/skema/program_analysis/CAST/matlab/variable_context.py b/skema/program_analysis/CAST/matlab/variable_context.py index 6ef6088964a..b2ab8af51b9 100644 --- a/skema/program_analysis/CAST/matlab/variable_context.py +++ b/skema/program_analysis/CAST/matlab/variable_context.py @@ -6,7 +6,6 @@ class VariableContext(object): def __init__(self): - """Docstring""" self.context = [{}] # Stack of context dictionaries self.context_return_values = [set()] # Stack of context return values self.all_symbols = {} @@ -78,11 +77,9 @@ def is_variable(self, symbol: str) -> bool: return symbol in self.all_symbols def get_node(self, symbol: str) -> Dict: - """Docstring""" return self.all_symbols[symbol]["node"] def get_type(self, symbol: str) -> str: - """Docstring""" return self.all_symbols[symbol]["type"] def update_type(self, symbol: str, type: str): @@ -92,22 +89,18 @@ def update_type(self, symbol: str, type: str): self.all_symbols[full_symbol_name]["type"] = type def add_return_value(self, symbol): - """Docstring""" self.context_return_values[-1].add(symbol) def remove_return_value(self, symbol): - """Docstring""" self.context_return_values[-1].discard(symbol) def generate_iterator(self): - """Docstring""" symbol = f"generated_iter_{self.iterator_id}" self.iterator_id += 1 return self.add_variable(symbol, "iterator", None) def generate_stop_condition(self): - """Docstring""" symbol = f"sc_{self.stop_condition_id}" self.stop_condition_id += 1 @@ -126,5 +119,4 @@ def set_internal(self): self.internal = True def unset_internal(self): - """Docstring""" self.internal = False From b3f443be9263ab1e3d6e52c9919d2da74e6ce8ae Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 14:22:19 -0700 Subject: [PATCH 52/84] Deleted an unused debug tree field --- skema/program_analysis/CAST/matlab/tree_out | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tree_out b/skema/program_analysis/CAST/matlab/tree_out index 4e52a78f6a7..c0352e3077a 100755 --- a/skema/program_analysis/CAST/matlab/tree_out +++ b/skema/program_analysis/CAST/matlab/tree_out @@ -11,10 +11,7 @@ from skema.program_analysis.tree_sitter_parsers.build_parsers import ( def print_tree(node: Node, indent = ''): """Display the node branch in pretty format""" for child in node.children: - if child.type == "\n": - # print(f"{indent} ") - pass - else: + if not child.type == "\n": print(f"{indent} {child.type}") print_tree(child, indent + ' ') From 7acbf48eb355b29eb2c836fd22dfdaa6eb5c5ae1 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 17:17:26 -0700 Subject: [PATCH 53/84] Fixed an indentation error --- skema/program_analysis/CAST/matlab/cast_out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skema/program_analysis/CAST/matlab/cast_out b/skema/program_analysis/CAST/matlab/cast_out index 5c196b19073..2d332dac26b 100755 --- a/skema/program_analysis/CAST/matlab/cast_out +++ b/skema/program_analysis/CAST/matlab/cast_out @@ -14,7 +14,7 @@ def remove_keys(json_obj, target_keys: List): """ remove all instances of the target keys from the json object""" def remove_key(json_obj, target_key): - """ remove all instances of the target key from the json object""" + """ remove all instances of the target key from the json object""" if isinstance(json_obj, dict): for target_key in target_keys: if target_key in json_obj.keys(): From 1c32a959b05b32ee20bb40d9cbca8a87077cef08 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 18:55:03 -0700 Subject: [PATCH 54/84] Added test cases for multi-element conditional blocks --- skema/program_analysis/CAST/matlab/tests/test_conditional.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py index dd73a25676f..63183c52824 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_conditional.py +++ b/skema/program_analysis/CAST/matlab/tests/test_conditional.py @@ -27,8 +27,10 @@ def test_if_else(): source = """ if x > 5 y = 6 + three = 3 else y = x + foo = 'bar' end """ @@ -37,8 +39,10 @@ def test_if_else(): assert isinstance(mi, ModelIf) assert_expression(mi.expr, op = ">", left = "x", right = "5") assert_assignment(mi.body[0], left="y", right = "6") + assert_assignment(mi.body[1], left="three", right = "3") # else assert_assignment(mi.orelse[0], left="y", right = "x") + assert_assignment(mi.orelse[1], left="foo", right = "'bar'") def test_if_elseif_else(): """ Test CAST from MATLAB 'if elseif else' conditional logic.""" From 80165e9176126eadd45797789d32cd8093075421 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 18:58:44 -0700 Subject: [PATCH 55/84] cleaned up conditional node handler --- .../CAST/matlab/matlab_to_cast.py | 84 ++++++++----------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index c0dcff05ea8..58fa0a33cc5 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -118,6 +118,8 @@ def run_old(self, root) -> List[Module]: def visit(self, node): """Switch execution based on node type""" # print(f"\nvisit {node.type}") + if node == None: + return None if node.type in ["program", "module", "source_file"] : return self.visit_module(node) @@ -153,12 +155,12 @@ def visit(self, node): return self.visit_do_loop_statement(node) elif node.type == "switch_statement": return self.visit_switch_statement(node) + elif node.type == "case_clause": + return self.visit_case_clause(node) + elif node.type == "otherwise_clause": + return self.visit_otherwise_clause(node) elif node.type == "if_statement": return self.visit_if_statement(node) - elif node.type == "elseif_clause": - return self.visit_elseif_clause(node) - elif node.type in ["else_clause", "otherwise_clause"]: - return self.visit_else_clause(node) elif node.type == "derived_type_definition": return self.visit_derived_type(node) elif node.type == "derived_type_member_expression": @@ -597,9 +599,10 @@ def visit_switch_statement(self, node): # get switch identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) - # get n case clauses + # get 0-n case clauses case_clauses = get_children_by_types(node, ["case_clause"]) + # get 0-1 otherwise clauses mi = self.visit(get_first_child_by_type(node, "otherwise_clause")) return mi @@ -607,6 +610,11 @@ def visit_switch_statement(self, node): def visit_if_statement(self, node): """ return a ModelIf if, elseif, and else clauses""" + # ModelIf ( + # expr: Expression + # body: list(body nodes) + # orelse: list(ModelIf) + # if_statement Tree-sitter syntax tree: # if # comparison_operator @@ -615,44 +623,35 @@ def visit_if_statement(self, node): # else_clause (0-1 of these) # end - # the initial ModelIf node is built just like the else-if clause - mi = self.visit_elseif_clause(node) + def model_if_node(_node): + """ return a ModelIf with comparison operator and body nodes. """ + ret = ModelIf() + # expression + ret.expr = self.visit(get_first_child_by_type( + _node, + "comparison_operator" + )) + # body + block = get_first_child_by_type(_node, "block") + if block: + ast_nodes = [self.visit(child) for child in block.children] + ret.body = [ast_node for ast_node in ast_nodes if ast_node] + return ret - # get 0-n elseif_clauses - elseif_clauses = get_children_by_types(node, ["elseif_clause"]) - for child in elseif_clauses: - elseif_node = self.visit(child) - if elseif_node: - if not mi.orelse: - mi.orelse = list() - mi.orelse.append(elseif_node) - - # get 0-1 else_clauses - else_clauses = get_children_by_types(node, ["else_clause"]) - for child in else_clauses: - else_node = self.visit(child) - if else_node.body: - for body_node in else_node.body: - if not mi.orelse: - mi.orelse = list() - mi.orelse.append(body_node) + # the if statement is defined by a ModelIf struct + mi = model_if_node(node) - return mi - - def visit_elseif_clause(self, node): - """ return a ModelIf with comparison operator and body nodes. """ - mi = self.visit_else_clause(node) - comparison_operator = get_first_child_by_type(node, "comparison_operator") - mi.expr = self.visit(comparison_operator) + # get 0-n elseif_clauses as ModelIf structs + elseif_clauses = get_children_by_types(node, ["elseif_clause"]) + mi.orelse = [model_if_node(child) for child in elseif_clauses] - return mi + # if an else clause exists, we add its ModelIf body directly + else_clauses = [get_first_child_by_type(node, "else_clause")] + for body in [model_if_node(e).body for e in else_clauses if e]: + mi.orelse += body - def visit_else_clause(self, node: Node): - """ Return a ModelIf with body nodes from Tree-sitter block nodes. """ - mi = ModelIf() - mi.body = self.get_body_nodes(node) return mi - + def visit_assignment(self, node): left, _, right = node.children @@ -1149,12 +1148,3 @@ def get_gromet_function_node(self, func_name: str) -> Name: return self.variable_context.get_node(func_name) return self.variable_context.add_variable(func_name, "function", None) - - def get_body_nodes(self, node): - """ Return valid body nodes from Tree-sitter block node. """ - block = get_first_child_by_type(node, "block") - ast_nodes = [self.visit(child) for child in block.children] - body_nodes = [ast_node for ast_node in ast_nodes if ast_node] - if len(body_nodes) > 0: - return body_nodes - return None From 07b67b4f273eb7c8d1a9b2e5ad232fab1ae6b630 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 19:22:58 -0700 Subject: [PATCH 56/84] Cleaned up comments --- .../CAST/matlab/matlab_to_cast.py | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 58fa0a33cc5..4bcbfae1b9f 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -608,13 +608,7 @@ def visit_switch_statement(self, node): return mi def visit_if_statement(self, node): - """ return a ModelIf if, elseif, and else clauses""" - - # ModelIf ( - # expr: Expression - # body: list(body nodes) - # orelse: list(ModelIf) - + """ return a node describing if, elseif, else conditional logic""" # if_statement Tree-sitter syntax tree: # if # comparison_operator @@ -623,34 +617,34 @@ def visit_if_statement(self, node): # else_clause (0-1 of these) # end - def model_if_node(_node): - """ return a ModelIf with comparison operator and body nodes. """ + def conditional(_node): + """ return a ModelIf struct for the conditional logic clause. """ ret = ModelIf() - # expression + # expression, possibly None ret.expr = self.visit(get_first_child_by_type( _node, "comparison_operator" )) - # body + # body, possibly None block = get_first_child_by_type(_node, "block") if block: ast_nodes = [self.visit(child) for child in block.children] ret.body = [ast_node for ast_node in ast_nodes if ast_node] return ret - # the if statement is defined by a ModelIf struct - mi = model_if_node(node) + # the if statement is returned as a ModelIf struct + ret = conditional(node) - # get 0-n elseif_clauses as ModelIf structs + # add 0-n elseif_clauses to the returned or-else list elseif_clauses = get_children_by_types(node, ["elseif_clause"]) - mi.orelse = [model_if_node(child) for child in elseif_clauses] + ret.orelse = [conditional(child) for child in elseif_clauses] - # if an else clause exists, we add its ModelIf body directly + # if an else clause exists, add its block nodes to the returned or-else list else_clauses = [get_first_child_by_type(node, "else_clause")] - for body in [model_if_node(e).body for e in else_clauses if e]: - mi.orelse += body + for body in [conditional(e).body for e in else_clauses]: + ret.orelse += body - return mi + return ret def visit_assignment(self, node): left, _, right = node.children From a0cee5897b30ecae04b95a191e6407a656679432 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 19:39:05 -0700 Subject: [PATCH 57/84] Fixed a list handling bug --- .../CAST/matlab/matlab_to_cast.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 4bcbfae1b9f..563d39aa4b1 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -612,20 +612,20 @@ def visit_if_statement(self, node): # if_statement Tree-sitter syntax tree: # if # comparison_operator - # body block with 1-n elements - # elseif_clause (0-n of these) - # else_clause (0-1 of these) + # block with 1-n instructions + # elseif_clause (0-n, with comparison_operator and instruction block) + # else_clause (0-1, instruction block only) # end def conditional(_node): """ return a ModelIf struct for the conditional logic clause. """ ret = ModelIf() - # expression, possibly None + # comparison_operator, possibly None ret.expr = self.visit(get_first_child_by_type( _node, "comparison_operator" )) - # body, possibly None + # instruction_block possibly None block = get_first_child_by_type(_node, "block") if block: ast_nodes = [self.visit(child) for child in block.children] @@ -635,12 +635,12 @@ def conditional(_node): # the if statement is returned as a ModelIf struct ret = conditional(node) - # add 0-n elseif_clauses to the returned or-else list + # add 0-n elseif_clauses to the returned ModelIf or-else list elseif_clauses = get_children_by_types(node, ["elseif_clause"]) ret.orelse = [conditional(child) for child in elseif_clauses] - # if an else clause exists, add its block nodes to the returned or-else list - else_clauses = [get_first_child_by_type(node, "else_clause")] + # 0-1 else clause, add instruction block to the ModelIf or-else list + else_clauses = get_children_by_types(node, ["else_clause"]) for body in [conditional(e).body for e in else_clauses]: ret.orelse += body From 93dd1149de92b858054376f49a48e76746a31be9 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 20:11:22 -0700 Subject: [PATCH 58/84] Added a test case ifor if-elseif conditional --- .../CAST/matlab/tests/test_conditional.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py index 63183c52824..694dad076be 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_conditional.py +++ b/skema/program_analysis/CAST/matlab/tests/test_conditional.py @@ -44,6 +44,27 @@ def test_if_else(): assert_assignment(mi.orelse[0], left="y", right = "x") assert_assignment(mi.orelse[1], left="foo", right = "'bar'") +def test_if_elseif(): + """ Test CAST from MATLAB 'if elseif else' conditional logic.""" + + source = """ + if x > 5 + y = 6 + elseif x > 0 + y = x + end + """ + + mi = cast_nodes(source)[0] + # if + assert isinstance(mi, ModelIf) + assert_expression(mi.expr, op = ">", left = "x", right = "5") + assert_assignment(mi.body[0], left="y", right = "6") + # elseif + assert isinstance(mi.orelse[0], ModelIf) + assert_expression(mi.orelse[0].expr, op = ">", left = "x", right = "0") + assert_assignment(mi.orelse[0].body[0], left="y", right = "x") + def test_if_elseif_else(): """ Test CAST from MATLAB 'if elseif else' conditional logic.""" From 53b79aedd7301c6f64ea6ca883b93bc7819eb55c Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 20:11:44 -0700 Subject: [PATCH 59/84] Cleaned up comments --- .../program_analysis/CAST/matlab/matlab_to_cast.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 563d39aa4b1..79fd86d0628 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -609,13 +609,6 @@ def visit_switch_statement(self, node): def visit_if_statement(self, node): """ return a node describing if, elseif, else conditional logic""" - # if_statement Tree-sitter syntax tree: - # if - # comparison_operator - # block with 1-n instructions - # elseif_clause (0-n, with comparison_operator and instruction block) - # else_clause (0-1, instruction block only) - # end def conditional(_node): """ return a ModelIf struct for the conditional logic clause. """ @@ -630,16 +623,15 @@ def conditional(_node): if block: ast_nodes = [self.visit(child) for child in block.children] ret.body = [ast_node for ast_node in ast_nodes if ast_node] + return ret # the if statement is returned as a ModelIf struct ret = conditional(node) - - # add 0-n elseif_clauses to the returned ModelIf or-else list + # add 0-n elseif_clauses elseif_clauses = get_children_by_types(node, ["elseif_clause"]) ret.orelse = [conditional(child) for child in elseif_clauses] - - # 0-1 else clause, add instruction block to the ModelIf or-else list + # add 0-1 else clause else_clauses = get_children_by_types(node, ["else_clause"]) for body in [conditional(e).body for e in else_clauses]: ret.orelse += body From c70fc38312f266e65b6dbb886b231928d9a868d4 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 6 Nov 2023 20:29:23 -0700 Subject: [PATCH 60/84] Added a guard for None values --- skema/program_analysis/CAST/matlab/matlab_to_cast.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 79fd86d0628..f2b727e311c 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -633,8 +633,9 @@ def conditional(_node): ret.orelse = [conditional(child) for child in elseif_clauses] # add 0-1 else clause else_clauses = get_children_by_types(node, ["else_clause"]) - for body in [conditional(e).body for e in else_clauses]: - ret.orelse += body + for block in [conditional(e).body for e in else_clauses]: + if block: + ret.orelse += block return ret From 449863893a0b1395fb7f622008dc8992cb6652c8 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 02:40:59 -0700 Subject: [PATCH 61/84] switch statement literal nodes --- .../CAST/matlab/matlab_to_cast.py | 94 +++++++++++++------ 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index f2b727e311c..aa0df79a5fc 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -155,10 +155,6 @@ def visit(self, node): return self.visit_do_loop_statement(node) elif node.type == "switch_statement": return self.visit_switch_statement(node) - elif node.type == "case_clause": - return self.visit_case_clause(node) - elif node.type == "otherwise_clause": - return self.visit_otherwise_clause(node) elif node.type == "if_statement": return self.visit_if_statement(node) elif node.type == "derived_type_definition": @@ -576,64 +572,100 @@ def visit_otherwise_clause(self, node): return mi def visit_switch_statement(self, node): - """ return a conditional logic statement based on the switch statement """ - # case 'one' - # n = 1; - # case {2, '2', 'two'} - # n = 2 - # otherwise - # n = 0; - # end - # - # The above statement can be reduced to: - # if s in ['one'] - # n = 1 - # elseif s in [2, '2', 'two'] - # n = 2 - # else - # n = 0 - # The first case clause becomes an if statement - # any subsequent case clauses become elseif statements - # a default clause, if one exists, becomes an else statement + """ return a conditional statement based on the switch statement """ + + literal_types = ["number", "string", "boolean", "array_literal"] + sequence_types = ["cell", "row"] + + def get_literals(literal_node, target_list): + # keep looking + sequences = get_children_by_types(literal_node, sequence_types) + for sequence in sequences: + get_literals(sequence, target_list) + # found + literals = get_children_by_types(literal_node, literal_types) + print(f"get_literals: Literals found: {len(literals)}") + ast_nodes = [self.visit(child) for child in literals] + target_list += [ast_node for ast_node in ast_nodes if ast_node] + return target_list + + + return Operator( + source_language="matlab", + interpreter=None, + version=None, + op=op, + operands=operands, + source_refs=[self.node_helper.get_source_ref(node)], + ) + def conditional(conditional_node, identifier): + """ Create a sequence of if-then conditionals """ + ret=ModelIf() + + # find target values + literals = get_literals(conditional_node, list()) + print(f"conditional: Literals found: {len(literals)}") + ret.orelse = literals + + # comparison operator + ret.expr = self.visit(get_first_child_by_type( + conditional_node, + "comparison_operator" + )) + + # instruction_block possibly None + block = get_first_child_by_type(conditional_node, "block") + if block: + ast_nodes = [self.visit(child) for child in block.children] + ret.body = [ast_node for ast_node in ast_nodes if ast_node] + + return ret # get switch identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) + # Create a placeholder ModelIf for now + ret = ModelIf() + # get 0-n case clauses case_clauses = get_children_by_types(node, ["case_clause"]) + ret.orelse = [conditional(child, identifier) for child in case_clauses] # get 0-1 otherwise clauses - mi = self.visit(get_first_child_by_type(node, "otherwise_clause")) + otherwise_clauses = get_children_by_types(node, ["otherwise_clause"]) + for block in [conditional(child, identifier).body for child in otherwise_clauses]: + if block: + ret.orelse += block - return mi + return ret def visit_if_statement(self, node): """ return a node describing if, elseif, else conditional logic""" - def conditional(_node): - """ return a ModelIf struct for the conditional logic clause. """ + def conditional(conditional_node): + """ return a ModelIf struct for the conditional logic node. """ ret = ModelIf() # comparison_operator, possibly None ret.expr = self.visit(get_first_child_by_type( - _node, + conditional_node, "comparison_operator" )) # instruction_block possibly None - block = get_first_child_by_type(_node, "block") + block = get_first_child_by_type(conditional_node, "block") if block: ast_nodes = [self.visit(child) for child in block.children] ret.body = [ast_node for ast_node in ast_nodes if ast_node] return ret - # the if statement is returned as a ModelIf struct + # the if statement is returned as a ModelIf AstNode ret = conditional(node) # add 0-n elseif_clauses elseif_clauses = get_children_by_types(node, ["elseif_clause"]) ret.orelse = [conditional(child) for child in elseif_clauses] # add 0-1 else clause else_clauses = get_children_by_types(node, ["else_clause"]) - for block in [conditional(e).body for e in else_clauses]: + for block in [conditional(child).body for child in else_clauses]: if block: ret.orelse += block From 6df4d2bcbe530465db43bfde2f1bc2dc95db581a Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 07:17:52 -0700 Subject: [PATCH 62/84] fixed conditional orelse logic, added switch statement --- .../CAST/matlab/matlab_to_cast.py | 107 ++++++++++-------- 1 file changed, 58 insertions(+), 49 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index aa0df79a5fc..caebce2206c 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -577,67 +577,67 @@ def visit_switch_statement(self, node): literal_types = ["number", "string", "boolean", "array_literal"] sequence_types = ["cell", "row"] - def get_literals(literal_node, target_list): - # keep looking - sequences = get_children_by_types(literal_node, sequence_types) - for sequence in sequences: - get_literals(sequence, target_list) - # found - literals = get_children_by_types(literal_node, literal_types) - print(f"get_literals: Literals found: {len(literals)}") - ast_nodes = [self.visit(child) for child in literals] - target_list += [ast_node for ast_node in ast_nodes if ast_node] - return target_list - + def get_literals(node, literals): + for child in node.children: + get_literals(child, literals + get_children_by_types(child, literal_types)) + return [self.visit(literal) for literal in literals] + + def get_operator(op, left, right): + """ Return a comparison operator between identifier and literal""" + return Operator( + source_language="matlab", + interpreter=None, + version=None, + op=op, + operands=[left, right] + # source_refs=[self.node_helper.get_source_ref(right)] + ) - return Operator( - source_language="matlab", - interpreter=None, - version=None, - op=op, - operands=operands, - source_refs=[self.node_helper.get_source_ref(node)], - ) - def conditional(conditional_node, identifier): + def conditional(conditional_node: Node, identifier): """ Create a sequence of if-then conditionals """ - ret=ModelIf() - - # find target values - literals = get_literals(conditional_node, list()) - print(f"conditional: Literals found: {len(literals)}") - ret.orelse = literals + mi=ModelIf() + + # there will either be a list containing a single literal_value + literal_values = get_children_by_types(conditional_node, literal_types) + ast_nodes = [self.visit(element) for element in literal_values] + valid_nodes = [ast_node for ast_node in ast_nodes if ast_node] + if len(valid_nodes) == 1: + mi.expr = get_operator("==", identifier, valid_nodes[0]) + + # else there will be a nested sequence containing them. + multiple_values = get_children_by_types(conditional_node, sequence_types) + # ... chain ifelse - # comparison operator - ret.expr = self.visit(get_first_child_by_type( - conditional_node, - "comparison_operator" - )) - # instruction_block possibly None + # instruction_block posibly None block = get_first_child_by_type(conditional_node, "block") if block: ast_nodes = [self.visit(child) for child in block.children] - ret.body = [ast_node for ast_node in ast_nodes if ast_node] + mi.body = [ast_node for ast_node in ast_nodes if ast_node] - return ret + return mi # get switch identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) - # Create a placeholder ModelIf for now - ret = ModelIf() - # get 0-n case clauses case_clauses = get_children_by_types(node, ["case_clause"]) - ret.orelse = [conditional(child, identifier) for child in case_clauses] + model_ifs = [conditional(child, identifier) for child in case_clauses] + + # link model_ifs as orelse lists + for i, model_if in enumerate(model_ifs[1:]): + model_ifs[i].orelse = [model_if] # get 0-1 otherwise clauses - otherwise_clauses = get_children_by_types(node, ["otherwise_clause"]) - for block in [conditional(child, identifier).body for child in otherwise_clauses]: + otherwise_clause = get_first_child_by_type(node, "otherwise_clause") + if otherwise_clause: + block = get_first_child_by_type(otherwise_clause, "block") if block: - ret.orelse += block + ast_nodes = [self.visit(child) for child in block.children] + last = model_ifs[len(model_ifs)-1] + last.orelse = [ast_node for ast_node in ast_nodes if ast_node] - return ret + return model_ifs[0] def visit_if_statement(self, node): """ return a node describing if, elseif, else conditional logic""" @@ -659,17 +659,26 @@ def conditional(conditional_node): return ret # the if statement is returned as a ModelIf AstNode - ret = conditional(node) + model_ifs = [conditional(node)] # add 0-n elseif_clauses elseif_clauses = get_children_by_types(node, ["elseif_clause"]) - ret.orelse = [conditional(child) for child in elseif_clauses] + model_ifs += [conditional(child) for child in elseif_clauses] + + # link model_ifs as orelse lists + for i, model_if in enumerate(model_ifs[1:]): + model_ifs[i].orelse = [model_if] + # add 0-1 else clause - else_clauses = get_children_by_types(node, ["else_clause"]) - for block in [conditional(child).body for child in else_clauses]: + else_clauses = [get_first_child_by_type(node, "else_clause")] + for else_clause in else_clauses: + block = get_first_child_by_type(else_clause, "block") if block: - ret.orelse += block + ast_nodes = [self.visit(child) for child in block.children] + last = model_ifs[len(model_ifs)-1] + last.orelse = [ast_node for ast_node in ast_nodes if ast_node] + + return model_ifs[0] - return ret def visit_assignment(self, node): left, _, right = node.children From 720353ed01865fecbfed0d61beb5e59106d2363b Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 07:32:13 -0700 Subject: [PATCH 63/84] else clause node bugfix --- skema/program_analysis/CAST/matlab/matlab_to_cast.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index caebce2206c..81fc72582f5 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -669,8 +669,8 @@ def conditional(conditional_node): model_ifs[i].orelse = [model_if] # add 0-1 else clause - else_clauses = [get_first_child_by_type(node, "else_clause")] - for else_clause in else_clauses: + else_clause = get_first_child_by_type(node, "else_clause") + if else_clause: block = get_first_child_by_type(else_clause, "block") if block: ast_nodes = [self.visit(child) for child in block.children] From 8fa103c2891edc81bba6775a65aa6fd910cf6c59 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 07:38:17 -0700 Subject: [PATCH 64/84] Fixed a bug in a test case --- skema/program_analysis/CAST/matlab/tests/test_conditional.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py index 694dad076be..6c8601f625d 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_conditional.py +++ b/skema/program_analysis/CAST/matlab/tests/test_conditional.py @@ -88,4 +88,4 @@ def test_if_elseif_else(): assert_expression(mi.orelse[0].expr, op = ">", left = "x", right = "0") assert_assignment(mi.orelse[0].body[0], left="y", right = "x") # else - assert_assignment(mi.orelse[1], left="y", right = "0") + assert_assignment(mi.orelse[0].orelse[0], left="y", right = "0") From 40d394f9396a76fb5547144039626c7b5d5724bb Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 07:54:08 -0700 Subject: [PATCH 65/84] Removed an outdated method, fixed comments --- .../CAST/matlab/matlab_to_cast.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 81fc72582f5..c0f93e4a15c 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -577,11 +577,6 @@ def visit_switch_statement(self, node): literal_types = ["number", "string", "boolean", "array_literal"] sequence_types = ["cell", "row"] - def get_literals(node, literals): - for child in node.children: - get_literals(child, literals + get_children_by_types(child, literal_types)) - return [self.visit(literal) for literal in literals] - def get_operator(op, left, right): """ Return a comparison operator between identifier and literal""" return Operator( @@ -597,16 +592,17 @@ def conditional(conditional_node: Node, identifier): """ Create a sequence of if-then conditionals """ mi=ModelIf() - # there will either be a list containing a single literal_value + # there will either be a list containing a single literal_value... literal_values = get_children_by_types(conditional_node, literal_types) ast_nodes = [self.visit(element) for element in literal_values] valid_nodes = [ast_node for ast_node in ast_nodes if ast_node] if len(valid_nodes) == 1: mi.expr = get_operator("==", identifier, valid_nodes[0]) - # else there will be a nested sequence containing them. + # ...or there will be a nested sequence containing them. multiple_values = get_children_by_types(conditional_node, sequence_types) - # ... chain ifelse + + # ... chain ifelse or test if value in list? # instruction_block posibly None @@ -660,15 +656,16 @@ def conditional(conditional_node): # the if statement is returned as a ModelIf AstNode model_ifs = [conditional(node)] + # add 0-n elseif_clauses elseif_clauses = get_children_by_types(node, ["elseif_clause"]) model_ifs += [conditional(child) for child in elseif_clauses] - # link model_ifs as orelse lists + # chain model_ifs as orelse lists for i, model_if in enumerate(model_ifs[1:]): model_ifs[i].orelse = [model_if] - # add 0-1 else clause + # add 0-1 else clause else_clause = get_first_child_by_type(node, "else_clause") if else_clause: block = get_first_child_by_type(else_clause, "block") From 37872cd1c7a9c20e6f210968ded3a6da1df71548 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 08:08:36 -0700 Subject: [PATCH 66/84] Added a test for switch statements --- .../CAST/matlab/tests/test_switch.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_switch.py b/skema/program_analysis/CAST/matlab/tests/test_switch.py index 34b51380226..f0b082c0e41 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_switch.py +++ b/skema/program_analysis/CAST/matlab/tests/test_switch.py @@ -1,11 +1,11 @@ from skema.program_analysis.CAST.matlab.tests.utils import ( - assert_var, + assert_assignment, assert_expression, cast_nodes ) -from skema.program_analysis.CAST2FN.model.cast import Assignment +from skema.program_analysis.CAST2FN.model.cast import ModelIf -def do_not_test_switch(): +def test_switch_single_values(): """ Test CAST from MATLAB switch statement.""" source = """ @@ -14,10 +14,25 @@ def do_not_test_switch(): n = 1; case 'two' n = 2; + x = y otherwise n = 0; end """ - # The root of the CAST should be Assignment - nodes = cast_nodes(source) + # The root of the CAST should a ModelIf instance + mi0 = cast_nodes(source)[0] + + # case 'one' + assert isinstance(mi0, ModelIf) + assert_assignment(mi0.body[0], left="n", right = "1") + assert_expression(mi0.expr, op="==", left = "s", right = "'one'") + + # case 'two' + mi1 = mi0.orelse[0] + assert isinstance(mi1, ModelIf) + assert_assignment(mi1.body[0], left="n", right = "2") + assert_assignment(mi1.body[1], left="x", right = "y") + + # otherwise + assert_assignment(mi1.orelse[0], left="n", right = "0") From 75b5b5a646b5253eddcc37021e09cfc3822b4562 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 16:13:45 -0700 Subject: [PATCH 67/84] Added helper methods --- skema/program_analysis/CAST/matlab/node_helper.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/skema/program_analysis/CAST/matlab/node_helper.py b/skema/program_analysis/CAST/matlab/node_helper.py index a76a5e91706..3ef9c3fc083 100644 --- a/skema/program_analysis/CAST/matlab/node_helper.py +++ b/skema/program_analysis/CAST/matlab/node_helper.py @@ -88,6 +88,21 @@ def get_first_child_index(node, type: str): if child.type == type: return i +def get_all(node: Node, types: List, found: List = list()): + """ return nodes with type in types in the entire node tree """ + for child in node.children: + if child.type in types: + found.append(child) + get_all( + node = child, + types = types, + found = found + ) + return found + +def valid(nodes): + """ return the node list without any None elements """ + return [node for node in nodes if node] def remove_comments(node: Node): """Remove comment nodes from tree-sitter parse tree""" From f95213b5f447abb636bcd51276a9891518a671d6 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 22:11:57 -0700 Subject: [PATCH 68/84] Added a node_helper method to recursively search a CAST node --- .../CAST/matlab/node_helper.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/node_helper.py b/skema/program_analysis/CAST/matlab/node_helper.py index 3ef9c3fc083..bda494b5f2b 100644 --- a/skema/program_analysis/CAST/matlab/node_helper.py +++ b/skema/program_analysis/CAST/matlab/node_helper.py @@ -88,17 +88,15 @@ def get_first_child_index(node, type: str): if child.type == type: return i -def get_all(node: Node, types: List, found: List = list()): - """ return nodes with type in types in the entire node tree """ - for child in node.children: - if child.type in types: - found.append(child) - get_all( - node = child, - types = types, - found = found - ) - return found +def get_all(node, types): + """ return all nodes with type in types from the entire node tree """ + def search(node, types, ret): + if node.type in types: + ret += [node] + for child in node.children: + search(child, types, ret) + return ret + return search(node, types, []) def valid(nodes): """ return the node list without any None elements """ From d0b281e03436e8b6191261083c8e1950ede44240 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 23:15:35 -0700 Subject: [PATCH 69/84] Added multi-value switch statement case clauses --- .../CAST/matlab/matlab_to_cast.py | 110 ++++++++++-------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index c0f93e4a15c..f48f4f42e6c 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -30,6 +30,7 @@ from skema.program_analysis.CAST.matlab.variable_context import VariableContext from skema.program_analysis.CAST.matlab.node_helper import ( + get_all, get_children_by_types, get_control_children, get_first_child_by_type, @@ -38,6 +39,7 @@ get_non_control_children, remove_comments, NodeHelper, + valid ) from skema.program_analysis.tree_sitter_parsers.build_parsers import INSTALLED_LANGUAGES_FILEPATH @@ -45,6 +47,9 @@ MATLAB_VERSION='matlab_version_here' class MatlabToCast(object): + + literal_types = ["number","string", "boolean", "array_literal"] + def __init__(self, source_path = "", source = ""): # if a source file path is provided, read source from file @@ -145,7 +150,7 @@ def visit(self, node): "math_expression", "relational_expression" ]: return self.visit_math_expression(node) - elif node.type in ["number", "array", "string", "boolean"]: + elif node.type in self.literal_types: return self.visit_literal(node) elif node.type == "keyword_statement": return self.visit_keyword_statement(node) @@ -564,74 +569,80 @@ def visit_do_loop_statement(self, node) -> Loop: source_refs=[self.node_helper.get_source_ref(node)], ) - def visit_otherwise_clause(self, node): - mi = ModelIf() - - # ... - - return mi def visit_switch_statement(self, node): """ return a conditional statement based on the switch statement """ + + # node types used for case comparison + case_node_types = self.literal_types + ["identifier"] - literal_types = ["number", "string", "boolean", "array_literal"] - sequence_types = ["cell", "row"] - + def get_value(ast_node): + """ return the value or var name of the CAST node """ + if isinstance(ast_node, Var): + return ast_node.val.name + return ast_node.value + + def multiple_case_values(cell_node): + """" return a literalValue List with literal values and var names """ + nodes = get_all(cell_node, case_node_types) + ast_nodes = valid([self.visit(node) for node in nodes]) + return LiteralValue( + value_type="List", + value = [get_value(ast_node) for ast_node in ast_nodes], + source_code_data_type=["matlab", MATLAB_VERSION, "unknown"] +# source_refs=[self.node_helper.get_source_ref(cell_node)] + ) + def get_operator(op, left, right): """ Return a comparison operator between identifier and literal""" return Operator( source_language="matlab", interpreter=None, - version=None, + version=MATLAB_VERSION, op=op, operands=[left, right] - # source_refs=[self.node_helper.get_source_ref(right)] ) - def conditional(conditional_node: Node, identifier): - """ Create a sequence of if-then conditionals """ - mi=ModelIf() - - # there will either be a list containing a single literal_value... - literal_values = get_children_by_types(conditional_node, literal_types) - ast_nodes = [self.visit(element) for element in literal_values] - valid_nodes = [ast_node for ast_node in ast_nodes if ast_node] - if len(valid_nodes) == 1: - mi.expr = get_operator("==", identifier, valid_nodes[0]) - - # ...or there will be a nested sequence containing them. - multiple_values = get_children_by_types(conditional_node, sequence_types) - - # ... chain ifelse or test if value in list? - - - # instruction_block posibly None - block = get_first_child_by_type(conditional_node, "block") + def get_expr(case_node, identifier): + """ return an operator for the case_node """ + # multiple case values + cell_node = get_first_child_by_type(case_node, "cell") + if (cell_node): + return get_operator("in", identifier, multiple_case_values(cell_node)) + # single case value + nodes = get_children_by_types(case_node, case_node_types) + ast_nodes = valid([self.visit(element) for element in nodes]) + return get_operator("==", identifier, ast_nodes[0]) + + def get_block(case_node): + """ return the instruction block for the case_node """ + block = get_first_child_by_type(case_node, "block") if block: - ast_nodes = [self.visit(child) for child in block.children] - mi.body = [ast_node for ast_node in ast_nodes if ast_node] + return valid([self.visit(child) for child in block.children]) + return None - return mi - - # get switch identifier + def model_if(case_node, identifier): + return ModelIf( + expr = get_expr(case_node, identifier), + body = get_block(case_node), + ) + + # switch statement identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) - - # get 0-n case clauses - case_clauses = get_children_by_types(node, ["case_clause"]) - model_ifs = [conditional(child, identifier) for child in case_clauses] - - # link model_ifs as orelse lists + + # 1-n case clauses linked with orelse logic + case_nodes = get_children_by_types(node, ["case_clause"]) + model_ifs=[model_if(case_node, identifier) for case_node in case_nodes] for i, model_if in enumerate(model_ifs[1:]): model_ifs[i].orelse = [model_if] - # get 0-1 otherwise clauses + # 0-1 otherwise clauses linked with else logic otherwise_clause = get_first_child_by_type(node, "otherwise_clause") if otherwise_clause: block = get_first_child_by_type(otherwise_clause, "block") if block: - ast_nodes = [self.visit(child) for child in block.children] last = model_ifs[len(model_ifs)-1] - last.orelse = [ast_node for ast_node in ast_nodes if ast_node] + last.orelse = valid([self.visit(child) for child in block.children]) return model_ifs[0] @@ -649,8 +660,7 @@ def conditional(conditional_node): # instruction_block possibly None block = get_first_child_by_type(conditional_node, "block") if block: - ast_nodes = [self.visit(child) for child in block.children] - ret.body = [ast_node for ast_node in ast_nodes if ast_node] + ret.body = valid([self.visit(child) for child in block.children]) return ret @@ -661,7 +671,7 @@ def conditional(conditional_node): elseif_clauses = get_children_by_types(node, ["elseif_clause"]) model_ifs += [conditional(child) for child in elseif_clauses] - # chain model_ifs as orelse lists + # chain model_ifs as orelse logic for i, model_if in enumerate(model_ifs[1:]): model_ifs[i].orelse = [model_if] @@ -670,12 +680,10 @@ def conditional(conditional_node): if else_clause: block = get_first_child_by_type(else_clause, "block") if block: - ast_nodes = [self.visit(child) for child in block.children] last = model_ifs[len(model_ifs)-1] - last.orelse = [ast_node for ast_node in ast_nodes if ast_node] + last.orelse = valid([self.visit(child) for child in block.children]) return model_ifs[0] - def visit_assignment(self, node): left, _, right = node.children From 3075e148248222de1ce3ebca63f6dec86b0fd163 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Tue, 7 Nov 2023 23:17:00 -0700 Subject: [PATCH 70/84] Added CI test case for switch case clauses with multiple values --- .../CAST/matlab/tests/test_switch.py | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_switch.py b/skema/program_analysis/CAST/matlab/tests/test_switch.py index f0b082c0e41..8561fd2076a 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_switch.py +++ b/skema/program_analysis/CAST/matlab/tests/test_switch.py @@ -14,25 +14,57 @@ def test_switch_single_values(): n = 1; case 'two' n = 2; - x = y + x = y; otherwise n = 0; end """ - # The root of the CAST should a ModelIf instance + # case clause 'one' mi0 = cast_nodes(source)[0] - - # case 'one' assert isinstance(mi0, ModelIf) assert_assignment(mi0.body[0], left="n", right = "1") assert_expression(mi0.expr, op="==", left = "s", right = "'one'") - # case 'two' + # case clause 'two' mi1 = mi0.orelse[0] assert isinstance(mi1, ModelIf) assert_assignment(mi1.body[0], left="n", right = "2") assert_assignment(mi1.body[1], left="x", right = "y") - # otherwise + # otherwise clause + assert_assignment(mi1.orelse[0], left="n", right = "0") + +def test_switch_multiple_values(): + """ Test CAST from MATLAB switch statement.""" + + source = """ + switch s + case {'one', 'two', 'three'} + n = 1; + case 2 + n = 2; + otherwise + n = 0; + end + """ + + # case clause {'one', 'two', 'three'} + mi0 = cast_nodes(source)[0] + assert isinstance(mi0, ModelIf) + assert_assignment(mi0.body[0], left="n", right = "1") + assert_expression( + mi0.expr, + op="in", + left = 's', + right = ["'one'", "'two'", "'three'"] + ) + + # case clause 2 + mi1 = mi0.orelse[0] + assert isinstance(mi1, ModelIf) + assert_assignment(mi1.body[0], left="n", right = "2") + assert_expression(mi1.expr, op="==", left = 's', right = 2) + + # otherwise clause assert_assignment(mi1.orelse[0], left="n", right = "0") From 0f9c3c7ecb703551345a74f3dcb243d2a4ac4d4d Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 8 Nov 2023 10:53:08 -0700 Subject: [PATCH 71/84] Fixed a typo in a test case --- skema/program_analysis/CAST/matlab/tests/test_switch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_switch.py b/skema/program_analysis/CAST/matlab/tests/test_switch.py index 8561fd2076a..bc4af8ea109 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_switch.py +++ b/skema/program_analysis/CAST/matlab/tests/test_switch.py @@ -64,7 +64,7 @@ def test_switch_multiple_values(): mi1 = mi0.orelse[0] assert isinstance(mi1, ModelIf) assert_assignment(mi1.body[0], left="n", right = "2") - assert_expression(mi1.expr, op="==", left = 's', right = 2) + assert_expression(mi1.expr, op="==", left = 's', right = "2") # otherwise clause assert_assignment(mi1.orelse[0], left="n", right = "0") From 09df370dc42402a091ca90899f69f2e780da8fa9 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 8 Nov 2023 11:41:36 -0700 Subject: [PATCH 72/84] cleaned up comments --- skema/program_analysis/CAST/matlab/matlab_to_cast.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index f48f4f42e6c..1529716fb3d 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -577,24 +577,24 @@ def visit_switch_statement(self, node): case_node_types = self.literal_types + ["identifier"] def get_value(ast_node): - """ return the value or var name of the CAST node """ + """ return the CAST node value or var name """ if isinstance(ast_node, Var): return ast_node.val.name return ast_node.value def multiple_case_values(cell_node): - """" return a literalValue List with literal values and var names """ + """" return a literalValue with List with values and var names """ nodes = get_all(cell_node, case_node_types) ast_nodes = valid([self.visit(node) for node in nodes]) return LiteralValue( value_type="List", value = [get_value(ast_node) for ast_node in ast_nodes], - source_code_data_type=["matlab", MATLAB_VERSION, "unknown"] -# source_refs=[self.node_helper.get_source_ref(cell_node)] + source_code_data_type=["matlab", MATLAB_VERSION, "unknown"], + source_refs=[self.node_helper.get_source_ref(cell_node)] ) def get_operator(op, left, right): - """ Return a comparison operator between identifier and literal""" + """ Return a comparison operator between identifier and value """ return Operator( source_language="matlab", interpreter=None, @@ -604,7 +604,7 @@ def get_operator(op, left, right): ) def get_expr(case_node, identifier): - """ return an operator for the case_node """ + """ return an Operator representing the case clause test """ # multiple case values cell_node = get_first_child_by_type(case_node, "cell") if (cell_node): From 2d1907318e0e6956ab18a9d517cdbab9b23990d1 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 8 Nov 2023 12:40:03 -0700 Subject: [PATCH 73/84] placeholder --- .../CAST/matlab/matlab_to_cast.py | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 1529716fb3d..481bd5286f0 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -582,8 +582,8 @@ def get_value(ast_node): return ast_node.val.name return ast_node.value - def multiple_case_values(cell_node): - """" return a literalValue with List with values and var names """ + def multiple_argument_case_operand(cell_node): + """" return a list of values for case comparison """ nodes = get_all(cell_node, case_node_types) ast_nodes = valid([self.visit(node) for node in nodes]) return LiteralValue( @@ -593,50 +593,55 @@ def multiple_case_values(cell_node): source_refs=[self.node_helper.get_source_ref(cell_node)] ) - def get_operator(op, left, right): - """ Return a comparison operator between identifier and value """ - return Operator( + def single_argument_case_operand(case_node): + """" return an ast_node for case comparison """ + nodes = get_children_by_types(case_node, case_node_types) + ast_nodes = valid([self.visit(element) for element in nodes]) + return ast_nodes[0] + + def get_case_operator(case_node, identifier): + """ return an Operator representing the case test """ + operator = Operator( source_language="matlab", interpreter=None, version=MATLAB_VERSION, - op=op, - operands=[left, right] + operands = [identifier] ) - - def get_expr(case_node, identifier): - """ return an Operator representing the case clause test """ - # multiple case values cell_node = get_first_child_by_type(case_node, "cell") if (cell_node): - return get_operator("in", identifier, multiple_case_values(cell_node)) - # single case value - nodes = get_children_by_types(case_node, case_node_types) - ast_nodes = valid([self.visit(element) for element in nodes]) - return get_operator("==", identifier, ast_nodes[0]) + # multiple case arguments + operator.op = "in" + operator.operands += [multiple_argument_case_operand(cell_node)] + else: + # single case argument + operator.op = "==" + operator.operands += [single_argument_case_operand(case_node)] + return operator - def get_block(case_node): - """ return the instruction block for the case_node """ + def get_case_instructions(case_node): + """ return the instruction block for the case """ block = get_first_child_by_type(case_node, "block") if block: return valid([self.visit(child) for child in block.children]) return None def model_if(case_node, identifier): + """ return conditional logic representing the case """ return ModelIf( - expr = get_expr(case_node, identifier), - body = get_block(case_node), + expr = get_case_operator(case_node, identifier), + body = get_case_instructions(case_node), ) # switch statement identifier identifier = self.visit(get_first_child_by_type(node, "identifier")) - # 1-n case clauses linked with orelse logic + # n case clauses as 'if then' nodes case_nodes = get_children_by_types(node, ["case_clause"]) model_ifs=[model_if(case_node, identifier) for case_node in case_nodes] for i, model_if in enumerate(model_ifs[1:]): model_ifs[i].orelse = [model_if] - # 0-1 otherwise clauses linked with else logic + # otherwise clause as 'else' node after last 'if then' node otherwise_clause = get_first_child_by_type(node, "otherwise_clause") if otherwise_clause: block = get_first_child_by_type(otherwise_clause, "block") @@ -652,12 +657,12 @@ def visit_if_statement(self, node): def conditional(conditional_node): """ return a ModelIf struct for the conditional logic node. """ ret = ModelIf() - # comparison_operator, possibly None + # comparison_operator ret.expr = self.visit(get_first_child_by_type( conditional_node, "comparison_operator" )) - # instruction_block possibly None + # instruction_block block = get_first_child_by_type(conditional_node, "block") if block: ret.body = valid([self.visit(child) for child in block.children]) @@ -667,11 +672,9 @@ def conditional(conditional_node): # the if statement is returned as a ModelIf AstNode model_ifs = [conditional(node)] - # add 0-n elseif_clauses + # add 0-n elseif clauses elseif_clauses = get_children_by_types(node, ["elseif_clause"]) model_ifs += [conditional(child) for child in elseif_clauses] - - # chain model_ifs as orelse logic for i, model_if in enumerate(model_ifs[1:]): model_ifs[i].orelse = [model_if] From f1f607bed86bf8a2562f5fc9ecdd76ec04842652 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 8 Nov 2023 13:26:50 -0700 Subject: [PATCH 74/84] changed variable names for consistency --- .../CAST/matlab/matlab_to_cast.py | 65 +++++++++---------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 481bd5286f0..b67c501e3cc 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -576,60 +576,53 @@ def visit_switch_statement(self, node): # node types used for case comparison case_node_types = self.literal_types + ["identifier"] - def get_value(ast_node): + def get_node_value(ast_node): """ return the CAST node value or var name """ if isinstance(ast_node, Var): return ast_node.val.name return ast_node.value - def multiple_argument_case_operand(cell_node): - """" return a list of values for case comparison """ - nodes = get_all(cell_node, case_node_types) - ast_nodes = valid([self.visit(node) for node in nodes]) - return LiteralValue( - value_type="List", - value = [get_value(ast_node) for ast_node in ast_nodes], - source_code_data_type=["matlab", MATLAB_VERSION, "unknown"], - source_refs=[self.node_helper.get_source_ref(cell_node)] + def get_operator(op, operands): + """ return an Operator representing the case test """ + return Operator( + source_language = "matlab", + interpreter = None, + version = MATLAB_VERSION, + op = op, + operands = operands ) - def single_argument_case_operand(case_node): - """" return an ast_node for case comparison """ - nodes = get_children_by_types(case_node, case_node_types) - ast_nodes = valid([self.visit(element) for element in nodes]) - return ast_nodes[0] - - def get_case_operator(case_node, identifier): + def get_case_expression(case_node, identifier): """ return an Operator representing the case test """ - operator = Operator( - source_language="matlab", - interpreter=None, - version=MATLAB_VERSION, - operands = [identifier] - ) cell_node = get_first_child_by_type(case_node, "cell") + # multiple case arguments if (cell_node): - # multiple case arguments - operator.op = "in" - operator.operands += [multiple_argument_case_operand(cell_node)] - else: - # single case argument - operator.op = "==" - operator.operands += [single_argument_case_operand(case_node)] - return operator + nodes = get_all(cell_node, case_node_types) + ast_nodes = valid([self.visit(node) for node in nodes]) + operand = LiteralValue( + value_type="List", + value = [get_node_value(node) for node in ast_nodes], + source_code_data_type=["matlab", MATLAB_VERSION, "unknown"], + source_refs=[self.node_helper.get_source_ref(cell_node)] + ) + return get_operator("in", [identifier, operand]) + # single case argument + nodes = get_children_by_types(case_node, case_node_types) + operand = valid([self.visit(node) for node in nodes])[0] + return get_operator("==", [identifier, operand]) - def get_case_instructions(case_node): + def get_case_body(case_node): """ return the instruction block for the case """ block = get_first_child_by_type(case_node, "block") if block: return valid([self.visit(child) for child in block.children]) return None - def model_if(case_node, identifier): + def get_model_if(case_node, identifier): """ return conditional logic representing the case """ return ModelIf( - expr = get_case_operator(case_node, identifier), - body = get_case_instructions(case_node), + expr = get_case_expression(case_node, identifier), + body = get_case_body(case_node), ) # switch statement identifier @@ -637,7 +630,7 @@ def model_if(case_node, identifier): # n case clauses as 'if then' nodes case_nodes = get_children_by_types(node, ["case_clause"]) - model_ifs=[model_if(case_node, identifier) for case_node in case_nodes] + model_ifs = [get_model_if(node, identifier) for node in case_nodes] for i, model_if in enumerate(model_ifs[1:]): model_ifs[i].orelse = [model_if] From 8c2c5c8a3b9c0286177e0f096ca3049b7a9ff7ef Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 8 Nov 2023 20:59:52 -0700 Subject: [PATCH 75/84] Renamed test file --- .../matlab/tests/{test_binary_operation.py => test_operators.oy} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename skema/program_analysis/CAST/matlab/tests/{test_binary_operation.py => test_operators.oy} (100%) diff --git a/skema/program_analysis/CAST/matlab/tests/test_binary_operation.py b/skema/program_analysis/CAST/matlab/tests/test_operators.oy similarity index 100% rename from skema/program_analysis/CAST/matlab/tests/test_binary_operation.py rename to skema/program_analysis/CAST/matlab/tests/test_operators.oy From 80717a4b8a0e9111a34ec5cb1e812f159e68705d Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 8 Nov 2023 21:00:23 -0700 Subject: [PATCH 76/84] Renamed file --- .../CAST/matlab/tests/{test_operators.oy => test_operators.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename skema/program_analysis/CAST/matlab/tests/{test_operators.oy => test_operators.py} (100%) diff --git a/skema/program_analysis/CAST/matlab/tests/test_operators.oy b/skema/program_analysis/CAST/matlab/tests/test_operators.py similarity index 100% rename from skema/program_analysis/CAST/matlab/tests/test_operators.oy rename to skema/program_analysis/CAST/matlab/tests/test_operators.py From e7bdf96eb688b8bdc7d6a5ef3297d1d79458bdee Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Wed, 8 Nov 2023 22:06:17 -0700 Subject: [PATCH 77/84] Added equality tests to the conditional logic test cases --- .../CAST/matlab/tests/test_operators.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_operators.py b/skema/program_analysis/CAST/matlab/tests/test_operators.py index 2496bdae13c..bd74e82d63b 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_operators.py +++ b/skema/program_analysis/CAST/matlab/tests/test_operators.py @@ -5,7 +5,7 @@ ) from skema.program_analysis.CAST2FN.model.cast import Assignment -def test_binary_operation(): +def test_binary_operator(): """ Test CAST from MATLAB binary operation statement.""" source = 'z = x + y' @@ -20,3 +20,19 @@ def test_binary_operation(): # right assignment operand is a binary expression assert_expression(nodes[0].right, op = "+", left = "x", right = "y") + +def do_not_test_unary_operator(): + """ Test CAST from MATLAB binary operation statement.""" + + source = 'z = -6' + + # cast nodes should be one assignment + nodes = cast_nodes(source) + assert len(nodes) == 1 + assert isinstance(nodes[0], Assignment) + + # Left assignment operand is the variable + assert_var(nodes[0].left, name = "z") + + # right assignment operand is a binary expression + assert_expression(nodes[0].right, op = "+", left = "x", right = "y") From 1f2ed08d490b3bdb8a33a25ea8fe42d7c3681c93 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Thu, 9 Nov 2023 00:26:02 -0700 Subject: [PATCH 78/84] organized CI tests --- .../CAST/matlab/tests/test_assignment.py | 34 ++++++++++++++----- .../CAST/matlab/tests/test_conditional.py | 12 +++---- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_assignment.py b/skema/program_analysis/CAST/matlab/tests/test_assignment.py index 08fd89af13a..7557a77cf43 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_assignment.py +++ b/skema/program_analysis/CAST/matlab/tests/test_assignment.py @@ -3,20 +3,38 @@ cast_nodes ) -# Test the CAST returned by processing the simplest MATLAB assignment +# Test assignment of different datatypes -def test_assignment(): - """ Test CAST from MATLAB 'assignment' statement.""" +def test_numeric(): + """ Test assignment of integer and real numbers.""" source = """ - x = 5 - y = "xxx" + x = 5; + y = 1.8; """ - # nodes should be two assignments nodes = cast_nodes(source) assert len(nodes) == 2 + + # integer assert_assignment(nodes[0], left = "x", right = "5") + # real + assert_assignment(nodes[1], left = "y", right = "1.8") + + +def test_string(): + """ Test assignment of strings. """ + + + source = """ + x = 'cat'; + y = "dog"; + """ + + nodes = cast_nodes(source) + assert len(nodes) == 2 - # When comparing strings you must include escaped quotes - assert_assignment(nodes[1], left = "y", right = "\"xxx\"") + # single quotes + assert_assignment(nodes[0], left = "x", right = "'cat'") + # double quotes + assert_assignment(nodes[1], left = "y", right = "\"dog\"") diff --git a/skema/program_analysis/CAST/matlab/tests/test_conditional.py b/skema/program_analysis/CAST/matlab/tests/test_conditional.py index 6c8601f625d..08a61f7265b 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_conditional.py +++ b/skema/program_analysis/CAST/matlab/tests/test_conditional.py @@ -9,7 +9,7 @@ def test_if(): """ Test CAST from MATLAB 'if' conditional logic.""" source = """ - if x > 5 + if x == 5 y = 6 end """ @@ -18,7 +18,7 @@ def test_if(): # if assert isinstance(mi, ModelIf) - assert_expression(mi.expr, op = ">", left = "x", right = "5") + assert_expression(mi.expr, op = "==", left = "x", right = "5") assert_assignment(mi.body[0], left="y", right = "6") def test_if_else(): @@ -48,9 +48,9 @@ def test_if_elseif(): """ Test CAST from MATLAB 'if elseif else' conditional logic.""" source = """ - if x > 5 + if x >= 5 y = 6 - elseif x > 0 + elseif x <= 0 y = x end """ @@ -58,11 +58,11 @@ def test_if_elseif(): mi = cast_nodes(source)[0] # if assert isinstance(mi, ModelIf) - assert_expression(mi.expr, op = ">", left = "x", right = "5") + assert_expression(mi.expr, op = ">=", left = "x", right = "5") assert_assignment(mi.body[0], left="y", right = "6") # elseif assert isinstance(mi.orelse[0], ModelIf) - assert_expression(mi.orelse[0].expr, op = ">", left = "x", right = "0") + assert_expression(mi.orelse[0].expr, op = "<=", left = "x", right = "0") assert_assignment(mi.orelse[0].body[0], left="y", right = "x") def test_if_elseif_else(): From 48f5c9e053b0cb6ac8d8f5e21797b4aa3510e0a5 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Thu, 9 Nov 2023 01:28:42 -0700 Subject: [PATCH 79/84] Added test cases for literal assignment --- .../CAST/matlab/tests/test_assignment.py | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/tests/test_assignment.py b/skema/program_analysis/CAST/matlab/tests/test_assignment.py index 7557a77cf43..2b38dd5702e 100644 --- a/skema/program_analysis/CAST/matlab/tests/test_assignment.py +++ b/skema/program_analysis/CAST/matlab/tests/test_assignment.py @@ -3,38 +3,43 @@ cast_nodes ) -# Test assignment of different datatypes +# Test CAST from assignment -def test_numeric(): - """ Test assignment of integer and real numbers.""" +def test_literal(): + """ Test assignment of literal types (number, string, boolean).""" source = """ - x = 5; - y = 1.8; + x = 5 + y = 1.8 + x = 'single' + y = "double" + yes = true + no = false """ nodes = cast_nodes(source) - assert len(nodes) == 2 + assert len(nodes) == 6 - # integer + # number assert_assignment(nodes[0], left = "x", right = "5") - # real assert_assignment(nodes[1], left = "y", right = "1.8") + # string + assert_assignment(nodes[2], left = "x", right = "'single'") + assert_assignment(nodes[3], left = "y", right = "\"double\"") + # boolean + assert_assignment(nodes[4], left = 'yes', right = 'true') + assert_assignment(nodes[5], left = 'no', right = 'false') -def test_string(): - """ Test assignment of strings. """ - +def test_identifier(): + """ Test assignment of identifiers.""" source = """ - x = 'cat'; - y = "dog"; + x = y """ nodes = cast_nodes(source) - assert len(nodes) == 2 + assert len(nodes) == 1 - # single quotes - assert_assignment(nodes[0], left = "x", right = "'cat'") - # double quotes - assert_assignment(nodes[1], left = "y", right = "\"dog\"") + # identifier + assert_assignment(nodes[0], left = 'x', right = 'y') From 9acaf6640d30692a034c88c52176d232f26dd7ce Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 13 Nov 2023 13:44:04 -0700 Subject: [PATCH 80/84] Added a missing source_refs node --- skema/program_analysis/CAST/matlab/matlab_to_cast.py | 1 + 1 file changed, 1 insertion(+) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index b67c501e3cc..0b8a5dbf2e8 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -623,6 +623,7 @@ def get_model_if(case_node, identifier): return ModelIf( expr = get_case_expression(case_node, identifier), body = get_case_body(case_node), + source_refs=[self.node_helper.get_source_ref(cell_node)] ) # switch statement identifier From 409d10d66ed16ba1faf0b4cd0100c3978a3831bc Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 13 Nov 2023 14:09:41 -0700 Subject: [PATCH 81/84] Added a missing source_refs node --- skema/program_analysis/CAST/matlab/matlab_to_cast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index 0b8a5dbf2e8..fe1e4c3c604 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -623,7 +623,7 @@ def get_model_if(case_node, identifier): return ModelIf( expr = get_case_expression(case_node, identifier), body = get_case_body(case_node), - source_refs=[self.node_helper.get_source_ref(cell_node)] + source_refs=[self.node_helper.get_source_ref(case_node)] ) # switch statement identifier From 5f64486a87e795eabf3b569a22cc25e250ff28d9 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 13 Nov 2023 14:22:27 -0700 Subject: [PATCH 82/84] Added missing source_refs node --- .../CAST/matlab/matlab_to_cast.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/skema/program_analysis/CAST/matlab/matlab_to_cast.py b/skema/program_analysis/CAST/matlab/matlab_to_cast.py index fe1e4c3c604..01305fff5f5 100644 --- a/skema/program_analysis/CAST/matlab/matlab_to_cast.py +++ b/skema/program_analysis/CAST/matlab/matlab_to_cast.py @@ -582,19 +582,21 @@ def get_node_value(ast_node): return ast_node.val.name return ast_node.value - def get_operator(op, operands): + def get_operator(op, operands, source_refs): """ return an Operator representing the case test """ return Operator( source_language = "matlab", interpreter = None, version = MATLAB_VERSION, op = op, - operands = operands + operands = operands, + source_refs = source_refs ) def get_case_expression(case_node, identifier): """ return an Operator representing the case test """ cell_node = get_first_child_by_type(case_node, "cell") + source_refs = self.node_helper.get_source_ref(case_node) # multiple case arguments if (cell_node): nodes = get_all(cell_node, case_node_types) @@ -605,11 +607,11 @@ def get_case_expression(case_node, identifier): source_code_data_type=["matlab", MATLAB_VERSION, "unknown"], source_refs=[self.node_helper.get_source_ref(cell_node)] ) - return get_operator("in", [identifier, operand]) + return get_operator("in", [identifier, operand], source_refs) # single case argument nodes = get_children_by_types(case_node, case_node_types) operand = valid([self.visit(node) for node in nodes])[0] - return get_operator("==", [identifier, operand]) + return get_operator("==", [identifier, operand], source_refs) def get_case_body(case_node): """ return the instruction block for the case """ @@ -650,12 +652,18 @@ def visit_if_statement(self, node): def conditional(conditional_node): """ return a ModelIf struct for the conditional logic node. """ - ret = ModelIf() + # comparison_operator - ret.expr = self.visit(get_first_child_by_type( + expr = self.visit(get_first_child_by_type( conditional_node, "comparison_operator" )) + + ret = ModelIf( + expr = expr, + source_refs=[self.node_helper.get_source_ref(conditional_node)] + ) + # instruction_block block = get_first_child_by_type(conditional_node, "block") if block: From 96b024312976fca22cf8ca07b75283da17e0d033 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 13 Nov 2023 14:26:02 -0700 Subject: [PATCH 83/84] Added CI tests for missing source_refs nodes --- skema/program_analysis/CAST/matlab/tests/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/skema/program_analysis/CAST/matlab/tests/utils.py b/skema/program_analysis/CAST/matlab/tests/utils.py index 68e1e65b0d1..d7474bdc3ad 100644 --- a/skema/program_analysis/CAST/matlab/tests/utils.py +++ b/skema/program_analysis/CAST/matlab/tests/utils.py @@ -11,12 +11,15 @@ def assert_var(var, name = ""): """ Test the Var for correct type and name. """ assert isinstance(var, Var) + assert not var.source_refs == None assert isinstance(var.val, Name) + assert not var.val.source_refs == None assert var.val.name == name def assert_literal_value(literal_value, value = ""): """ Test the LiteralValue for correct type and value. """ assert isinstance(literal_value, LiteralValue) + assert not literal_value.source_refs == None assert literal_value.value == value def assert_operand(operand, value = ""): @@ -31,12 +34,14 @@ def assert_operand(operand, value = ""): def assert_assignment(assignment, left = "", right = ""): """ Test an Assignment for correct type and operands. """ assert isinstance(assignment, Assignment) + assert not assignment.source_refs == None assert_operand(assignment.left, left) assert_operand(assignment.right, right) def assert_expression(expression, op = "", left = "", right = ""): """ Test an Operator for correct type, operation, and operands. """ assert isinstance(expression, Operator) + assert not expression.source_refs == None assert expression.op == op assert_operand(expression.operands[0], left) assert_operand(expression.operands[1], right) From 3be159517476075ecb5d5545e2b2ad079c3ffd46 Mon Sep 17 00:00:00 2001 From: Joseph Astier Date: Mon, 13 Nov 2023 14:30:13 -0700 Subject: [PATCH 84/84] Added testing for null source_refs nodes --- skema/program_analysis/CAST/matlab/tests/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skema/program_analysis/CAST/matlab/tests/utils.py b/skema/program_analysis/CAST/matlab/tests/utils.py index d7474bdc3ad..b145497b19c 100644 --- a/skema/program_analysis/CAST/matlab/tests/utils.py +++ b/skema/program_analysis/CAST/matlab/tests/utils.py @@ -8,6 +8,8 @@ Var ) +# Tests now check for null source_refs nodes + def assert_var(var, name = ""): """ Test the Var for correct type and name. """ assert isinstance(var, Var)