crawl_sourcecode.py

""" This script can be used to test the equivalence in parsing between
rustpython and cpython.

Usage example:

$ python crawl_sourcecode.py crawl_sourcecode.py > cpython.txt
$ cargo run crawl_sourcecode.py crawl_sourcecode.py > rustpython.txt
$ diff cpython.txt rustpython.txt
"""


import ast
import sys
import symtable
import dis

filename = sys.argv[1]
print('Crawling file:', filename)


with open(filename, 'r') as f:
    source = f.read()

t = ast.parse(source)
print(t)

shift = 3
def print_node(node, indent=0):
    indents = ' ' * indent
    if isinstance(node, ast.AST):
        lineno = 'row={}'.format(node.lineno) if hasattr(node, 'lineno') else ''
        print(indents, "NODE", node.__class__.__name__, lineno)
        for field in node._fields:
            print(indents,'-', field)
            f = getattr(node, field)
            if isinstance(f, list):
                for f2 in f:
                    print_node(f2, indent=indent+shift)
            else:
                print_node(f, indent=indent+shift)
    else:
        print(indents, 'OBJ', node)

print_node(t)

# print(ast.dump(t))
flag_names = [
    'is_referenced',
    'is_assigned',
    'is_global',
    'is_local',
    'is_parameter',
    'is_free',
]

def print_table(table, indent=0):
    indents = ' ' * indent
    print(indents, 'table:', table.get_name())
    print(indents, ' ', 'name:', table.get_name())
    print(indents, ' ', 'type:', table.get_type())
    print(indents, ' ', 'line:', table.get_lineno())
    print(indents, ' ', 'identifiers:', table.get_identifiers())
    print(indents, ' ', 'Syms:')
    for sym in table.get_symbols():
        flags = []
        for flag_name in flag_names:
            func = getattr(sym, flag_name)
            if func():
                flags.append(flag_name)
        print(indents, '   sym:', sym.get_name(), 'flags:', ' '.join(flags))
    if table.has_children():
        print(indents, ' ', 'Child tables:')
        for child in table.get_children():
            print_table(child, indent=indent+shift)

table = symtable.symtable(source, 'a', 'exec')
print_table(table)

print()
print('======== dis.dis ========')
print()
co = compile(source, filename, 'exec')
dis.dis(co)