Port to Python3

.gitignore

@@ -0,0 +1,4 @@
+venv/
+venv2/
+.vscode/
+results/

CONTRIBUTORS

@@ -3,3 +3,5 @@ A big thank you to everyone who is helping out with this project.
 Below is a list of contributors:
 
 Rongkuan Ma - University of Sheffield
+
+Hammond Pearce - NYU

INSTALL.rst

@@ -1,11 +1,11 @@
-``ICSREF`` is developed in Python 2.7. To install it on a fresh Ubuntu 16.04 LTS system you can follow these steps:
+``ICSREF`` is developed in Python 3+. To install it on a fresh Ubuntu 20.04 LTS system you can follow these steps:
 
 Install the system dependencies:
 --------------------------------
 
 .. code-block:: none
 
-    sudo apt install git python-pip libcapstone3 python-dev libffi-dev build-essential virtualenvwrapper graphviz libgraphviz-dev graphviz-dev pkg-config
+    sudo apt install git python3-pip libcapstone3 python3-dev libffi-dev build-essential graphviz libgraphviz-dev graphviz-dev pkg-config
 
 Install radare2
 ---------------
@@ -27,16 +27,18 @@ Create a virtual environment (in a fresh shell)
 
 .. code-block:: none
 
-    mkvirtualenv icsref
+    virtualenv venv
+    source venv/bin/activate
+
 
 Install the python package dependencies
 ---------------------------------------
 
-To ensure stability, local wheels of the dependencies are included in the ``wheelhouse`` directory:
-
 .. code-block:: none
 
-    pip install --no-index --find-links=wheelhouse -r requirements.txt
+    pip install -r requirements_full.txt
+    OR (sometimes the above fails)
+    pip install networkx r2pipe dill ujson cmd2 angr pygraphviz pymodbus testtools six==1.14.0
 
 Create bash alias
 -----------------

icsref/PRG_analysis.py

@@ -15,6 +15,7 @@
 import dill
 import struct
 import logging
+import binascii
 from difflib import SequenceMatcher
 from glob import glob
 
@@ -164,14 +165,14 @@ def __find_blocks(self):
         """
 
         # Matches the prologue
-        prologue = '\x0d\xc0\xa0\xe1\x00\x58\x2d\xe9\x0c\xb0\xa0\xe1'
+        prologue = b'\x0d\xc0\xa0\xe1\x00\x58\x2d\xe9\x0c\xb0\xa0\xe1'
         beginnings = self.__allindices(self.hexdump, prologue)
         # Matches the epilogue
-        epilogue = '\x00\xa8\x1b\xe9'
+        epilogue = b'\x00\xa8\x1b\xe9'
         endings = self.__allindices(self.hexdump, epilogue)
         endings = [i+4 for i in endings]
 
-        return zip(beginnings, endings)
+        return list(zip(beginnings, endings))
 
     def __find_functions(self):
         """
@@ -182,6 +183,7 @@ def __find_functions(self):
         # Set r2 architecture configuration - Processor specific
         r2.cmd('e asm.arch=arm; e asm.bits=32; e cfg.bigendian=false')
         # Instantiate Functions
+        #functions = list(self.FunctionBoundaries)
         for i in range(len(self.FunctionBoundaries)):
             # Code disassembly
             # Start: MOV, STMFD, MOV
@@ -218,28 +220,29 @@ def __find_dynlibs(self):
         """
         offset = self.dynlib_end
         # Reverse find 0xFFFF (offset for the beginning of strings)
-        dynlib_offset = self.hexdump.rfind('\xff\xff',0,offset) + 2
+        dynlib_offset = self.hexdump.rfind(b'\xff\xff',0,offset) + 2
         dynlibs = {}
         # Match printable ASCII characters
-        dynlib = re.search('[ -~]*', self.hexdump[dynlib_offset:]).group(0)
+        dynlib = re.search(b'[ -~]*', self.hexdump[dynlib_offset:]).group(0)
         # Find the offsets to dynamic libs
         while dynlib:
             dynlib_offset += len(dynlib) + 1
-            temp = self.hexdump[dynlib_offset:dynlib_offset+2].encode('hex')
+            temp = binascii.hexlify(self.hexdump[dynlib_offset:dynlib_offset+2])
+            temp = temp.decode('ascii') #this might be all kinds of weird
             jump_offset = int(''.join([m[2:4]+m[0:2] for m in [temp[i:i+4] for i in range(0,len(temp),4)]]),16) * 4 + 8
             dynlibs[jump_offset] = dynlib
             dynlib_offset += 2
-            dynlib = re.search('[ -~]*', self.hexdump[dynlib_offset:]).group(0)
+            dynlib = re.search(b'[ -~]*', self.hexdump[dynlib_offset:]).group(0)
         return dynlibs
 
     def __find_statlibs(self):
         entry_offset = self.Functions[-1].start
         stop_offset  = self.FunctionBoundaries[-1][1]-8
         funs = [x for x, _ in self.FunctionBoundaries]
         # Change 0x2000 location of writing address in OUTRO with 0x10000000 to not overwrite code
-        code_start = '\x00\x20\x00\x00'
-        with open('temphexdump.bin', 'w') as f:
-            hexdump_mod = self.hexdump.replace(code_start, '\x00\x00\x00\x10')
+        code_start = b'\x00\x20\x00\x00'
+        with open('temphexdump.bin', 'wb') as f:
+            hexdump_mod = self.hexdump.replace(code_start, b'\x00\x00\x00\x10')
             f.write(hexdump_mod)
         proj = angr.Project('temphexdump.bin', load_options={'main_opts': {'backend': 'blob', 'custom_base_addr': 0, 'custom_arch':'ARMEL', 'custom_entry_point':0x50}, 'auto_load_libs':False})
         state = proj.factory.entry_state()
@@ -307,7 +310,7 @@ def __save_object(self):
                 raise
         dat_f = os.path.join(path, '{}_init_analysis.dat'.format(self.name))
 
-        with open(dat_f, 'w') as f:
+        with open(dat_f, 'wb') as f:
             dill.dump(self, f)
 
     def __allindices(self, file_bytes, sub, offset=0):
@@ -317,7 +320,7 @@ def __allindices(self, file_bytes, sub, offset=0):
         :param: file_bytes: bytes to perform the search on
         :param: sub: substring to search for in file_bytes
         """
-
+        
         i = file_bytes.find(sub, offset)
         listindex=[]
         while i >= 0:
@@ -330,7 +333,7 @@ def __strings(self):
         Finds consecutive <= 4-byte ASCII character strings
         """
         strings = {}
-        p=re.compile('([ -~]{4,})')
+        p=re.compile(b'([ -~]{4,})')
         for m in p.finditer(self.hexdump):
             strings[m.start()] = m.group()
         return strings
@@ -379,7 +382,7 @@ def __init__(self, path, start, stop, hexdump, disasm):
             if len(op) < 6:
                 op_str += line[43:].split(' ')[0]
         # Function opcodes SHA256 hash
-        self.hash = hashlib.sha256(op_str).hexdigest()
+        self.hash = hashlib.sha256(op_str.encode()).hexdigest()
         # Initialize list of calls from function. Gets populated later
         self.calls = {}
 

requirements.txt

@@ -1,9 +1,10 @@
-networkx==1.9
-r2pipe==0.9.5
-dill==0.2.7.1
-ujson==1.35
-cmd2==0.7.9
-angr==7.7.9.14
-pygraphviz==1.3.1
-pymodbus==1.4.0
-testtools==2.3.0
+networkx
+r2pipe
+dill
+ujson
+cmd2
+angr
+pygraphviz
+pymodbus
+testtools
+six==1.14.0

requirements_full.txt

@@ -0,0 +1,75 @@
+ailment==8.20.7.27
+angr==8.20.7.27
+appdirs==1.4.3
+archinfo==8.20.7.27
+attrs==20.2.0
+bitstring==3.1.7
+CacheControl==0.12.6
+cachetools==4.1.1
+capstone==4.0.2
+certifi==2019.11.28
+cffi==1.14.2
+chardet==3.0.4
+claripy==8.20.7.27
+cle==8.20.7.27
+cmd2==1.3.9
+colorama==0.4.3
+contextlib2==0.6.0
+CppHeaderParser==2.7.4
+decorator==4.4.2
+dill==0.3.2
+distlib==0.3.0
+distro==1.4.0
+dpkt==1.9.3
+extras==1.0.0
+fixtures==3.0.0
+future==0.18.2
+gitdb==4.0.5
+GitPython==3.1.8
+html5lib==1.0.1
+idna==2.8
+ipaddr==2.2.0
+itanium-demangler==1.0
+linecache2==1.0.0
+lockfile==0.12.2
+msgpack==0.6.2
+mulpyplexer==0.8
+networkx==2.5
+packaging==20.3
+pbr==5.5.0
+pefile==2019.4.18
+pep517==0.8.2
+plumbum==1.6.9
+ply==3.11
+progress==1.5
+progressbar2==3.53.1
+protobuf==3.13.0
+psutil==5.7.2
+pycparser==2.20
+pyelftools==0.26
+pygraphviz==1.6
+pymodbus==2.3.0
+pyparsing==2.4.6
+pyperclip==1.8.0
+pyserial==3.4
+PySMT==0.9.0
+python-mimeparse==1.6.0
+python-utils==2.4.0
+pytoml==0.1.21
+pyvex==8.20.7.27
+r2pipe==1.4.2
+requests==2.22.0
+retrying==1.3.3
+rpyc==4.1.5
+six==1.14.0
+smmap==3.0.4
+sortedcontainers==2.2.2
+testtools==2.4.0
+traceback2==1.4.0
+ujson==3.2.0
+unicorn==1.0.2rc4
+unittest2==1.1.0
+urllib3==1.25.8
+wcwidth==0.2.5
+webencodings==0.5.1
+z3-solver==4.8.8.0