Initial pre-commit additions and cleanup. mypy compatibility fixes

Add an initial pre-commit pipeline and run across all files in the repo.
Add various fixes for mypy related errors including adding stubs,
ignoring certain dependencies in setup.cfg (for those where stubs
couldn't be located), and manually add type ignore comments where
necessary. These mypy related issues should be revisited in the future
to see if they can be cleaned up in a more coherent way.

.pre-commit-config.yaml

@@ -0,0 +1,86 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-merge-conflict
+    -   id: debug-statements
+
+-   repo: https://github.com/asottile/reorder_python_imports
+    rev: v2.6.0
+    hooks:
+    -   id: reorder-python-imports
+        files: ^src/|test/
+
+-   repo: local
+    hooks:
+    -   id: mypy
+        name: mypy
+        entry: mypy ait
+        language: system
+        pass_filenames: false
+        exclude: "bin/"
+
+-   repo: local
+    hooks:
+    -   id: black
+        name: black
+        entry: black
+        files: ^src/|test/
+        language: system
+        #exclude_types: [yaml, yml, dat, txt]
+        types: [python]
+
+-   repo: local
+    hooks:
+    -   id: flake8
+        name: flake8
+        entry: flake8 ait
+        language: system
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: tlm_yaml_check
+        name: AIT TLM YAML check
+        entry: 'ait-yaml-validate --tlm'
+        language: system
+        stages: [push]
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: cmd_yaml_check
+        name: AIT CMD YAML check
+        entry: 'ait-yaml-validate --cmd'
+        language: system
+        stages: [push]
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: evr_yaml_check
+        name: AIT EVR YAML check
+        entry: 'ait-yaml-validate --evr'
+        language: system
+        stages: [push]
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: limits_yaml_check
+        name: AIT LIMITS YAML check
+        entry: 'ait-yaml-validate --limits'
+        language: system
+        stages: [push]
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: tests
+        name: Tests
+        entry: pytest
+        language: system
+        stages: [push]
+        pass_filenames: false

README.rst

@@ -5,7 +5,7 @@ The AMMOS Instrument Toolkit (Formerly the Bespoke Links to Instruments
 for Surface and Space (BLISS)) is a Python-based software suite
 developed to handle Ground Data System (GDS), Electronic Ground Support
 Equipment (EGSE), commanding, telemetry uplink/downlink, and sequencing
-for instrument and CubeSat Missions. It is a generalization and expansion 
+for instrument and CubeSat Missions. It is a generalization and expansion
 of tools developed for a number of ISS
 missions.
 
@@ -39,7 +39,7 @@ Guide <https://github.com/NASA-AMMOS/AIT-Core/wiki/Contributing>`__.
 |docs|
 
 .. |travis| image:: https://travis-ci.com/NASA-AMMOS/AIT-Core.svg?branch=master
-    :target: https://travis-ci.com/NASA-AMMOS/AIT-Core 
+    :target: https://travis-ci.com/NASA-AMMOS/AIT-Core
 
 .. |docs| image:: https://readthedocs.org/projects/ait-core/badge/?version=master
     :alt: Documentation Status

ait/core/__init__.py

@@ -28,19 +28,19 @@ def deprecated_func(*args, **kwargs):
     return deprecated_decorator
 
 
-sys.modules["ait"].deprecated = deprecated
-sys.modules["ait"].DEFAULT_CMD_PORT = 3075
-sys.modules["ait"].DEFAULT_CMD_HOST = "127.0.0.1"
+sys.modules["ait"].deprecated = deprecated  # type: ignore[attr-defined]
+sys.modules["ait"].DEFAULT_CMD_PORT = 3075  # type: ignore[attr-defined]
+sys.modules["ait"].DEFAULT_CMD_HOST = "127.0.0.1"  # type: ignore[attr-defined]
 
 ## Name of the ZMQ topic used to accept commands from external sources
-sys.modules["ait"].DEFAULT_CMD_TOPIC = "__commands__"
+sys.modules["ait"].DEFAULT_CMD_TOPIC = "__commands__"  # type: ignore[attr-defined]
 
 ## Name of the ZMQ topic / stream used for making telemetry packets available to the script API
-sys.modules["ait"].DEFAULT_TLM_TOPIC = "__tlmpkts__"
+sys.modules["ait"].DEFAULT_TLM_TOPIC = "__tlmpkts__"  # type: ignore[attr-defined]
 
 ## Number of seconds to sleep after ZmqSocket.connect() call, affects clients
-sys.modules["ait"].DEFAULT_CMD_ZMQ_SLEEP = 1
+sys.modules["ait"].DEFAULT_CMD_ZMQ_SLEEP = 1  # type: ignore[attr-defined]
 
 
-sys.modules["ait"].SERVER_DEFAULT_XSUB_URL = "tcp://*:5559"
-sys.modules["ait"].SERVER_DEFAULT_XPUB_URL = "tcp://*:5560"
+sys.modules["ait"].SERVER_DEFAULT_XSUB_URL = "tcp://*:5559"  # type: ignore[attr-defined]
+sys.modules["ait"].SERVER_DEFAULT_XPUB_URL = "tcp://*:5560"  # type: ignore[attr-defined]

ait/core/bin/ait_cmd_send.py

@@ -27,7 +27,7 @@
 """
 usage: ait-cmd-send [options] command [arguments]
 
-Sends the given command and its arguments to the ISS simulator via 
+Sends the given command and its arguments to the ISS simulator via
 the AIT server, or if the 'udp' flag is set then directly via UDP.
 
 --verbose          Hexdump data                  (default: False)

ait/core/bin/ait_tlm_simulate.py

@@ -15,7 +15,7 @@
 # information to foreign countries or providing access to foreign persons.
 
 """
-Usage: ait-tlm-simulate [options] 
+Usage: ait-tlm-simulate [options]
 
 Sends simulated telemetry.
 
@@ -25,11 +25,11 @@
   --packetFill        Byte to fill packet with         (default:None)
 
 If no packetName specified, will choose the first available packetDefn.
-If no packetFill specified, will fill packet using range. 
+If no packetFill specified, will fill packet using range.
 
 Examples:
 
-  $ ait-tlm-simulate 
+  $ ait-tlm-simulate
 
 """
 

ait/core/bsc.py

@@ -39,16 +39,16 @@
 
 RAW_SOCKET_FD = None
 try:
-    import rawsocket
+    import rawsocket  # type: ignore
 
     RAW_SOCKET_FD = rawsocket.rawsocket_fd()
 except ImportError:
-    log.debug(
+    log.debug(  # type: ignore
         "The rawsocket library cannot be imported. "
         "Defaulting to the non-rawsocket approach."
     )
 except IOError:
-    log.info(
+    log.info(  # type: ignore
         "Unable to spawn rawsocket-helper. "
         "This may be a permissions issue (not SUID root?). "
         "Defaulting to non-rawsocket approach."

ait/core/cfg.py

@@ -229,7 +229,7 @@ class AitConfig(object):
     _ROOT_DIR = os.path.abspath(os.environ.get("AIT_ROOT", os.getcwd()))
 
     if "AIT_ROOT" not in os.environ:
-        log.warn('AIT_ROOT not set.  Defaulting to "%s"' % _ROOT_DIR)
+        log.warn('AIT_ROOT not set.  Defaulting to "%s"' % _ROOT_DIR)  # type: ignore
 
     def __init__(self, filename=None, data=None, config=None, pathvars=None):
         """Creates a new AitConfig object with configuration data read from
@@ -424,7 +424,7 @@ def addPathVariables(self, pathvars):
 
 
 # Create a singleton AitConfig accessible via ait.config
-sys.modules["ait"].config = AitConfig()
+sys.modules["ait"].config = AitConfig()  # type: ignore[attr-defined]
 
 # Re-initialize logging now that ait.config.logging.* parameters may exist.
 log.reinit()

ait/core/cmd.py

@@ -287,7 +287,7 @@ def __init__(self, *args, **kwargs):
             setattr(self, slot, kwargs.get(name, None))
 
         if self.ccsds:
-            import ccsds
+            import ait.core.ccsds as ccsds
 
             self.ccsds = ccsds.CcsdsDefinition(**self.ccsds)
 

ait/core/data/table_schema.json

@@ -77,4 +77,3 @@
         }
     }
 }
-

ait/core/db.py

@@ -36,7 +36,7 @@
 Backend = None
 
 
-@ait.deprecated(
+@ait.deprecated(  # type: ignore
     "connect() will be replaced with SQLiteBackend methods in a future release"
 )
 def connect(database):
@@ -47,7 +47,7 @@ def connect(database):
     return Backend.connect(database)
 
 
-@ait.deprecated(
+@ait.deprecated(  # type: ignore
     "create() will be replaced with SQLiteBackend methods in a future release"
 )
 def create(database, tlmdict=None):
@@ -65,7 +65,7 @@ def create(database, tlmdict=None):
     return dbconn
 
 
-@ait.deprecated(
+@ait.deprecated(  # type: ignore
     "createTable() will be replaced with SQLiteBackend methods in a future release"
 )
 def createTable(dbconn, pd):
@@ -77,7 +77,7 @@ def createTable(dbconn, pd):
     dbconn.commit()
 
 
-@ait.deprecated(
+@ait.deprecated(  # type: ignore
     "getTypename() will be replaced with SQLiteBackend methods in a future release"
 )
 def getTypename(defn):
@@ -86,7 +86,7 @@ def getTypename(defn):
     return "REAL" if defn.type.float or defn.dntoeu else "INTEGER"
 
 
-@ait.deprecated(
+@ait.deprecated(  # type: ignore
     "insert() will be replaced with SQLiteBackend methods in a future release"
 )
 def insert(dbconn, packet):
@@ -111,7 +111,7 @@ def insert(dbconn, packet):
     dbconn.execute(sql, values)
 
 
-@ait.deprecated("use() will be replaced with SQLiteBackend methods in a future release")
+@ait.deprecated("use() will be replaced with SQLiteBackend methods in a future release")  # type: ignore
 def use(backend):
     """Use the given database backend, e.g. 'MySQLdb', 'psycopg2',
     'MySQLdb', etc.
@@ -125,8 +125,8 @@ def use(backend):
         raise cfg.AitConfigError(msg)
 
 
-if ait.config.get("database.backend"):
-    use(ait.config.get("database.backend"))
+if ait.config.get("database.backend"):  # type: ignore
+    use(ait.config.get("database.backend"))  # type: ignore
 
 
 class AITDBResult:
@@ -264,7 +264,7 @@ class GenericBackend(object):
 
     __metaclass__ = ABCMeta
 
-    _backend = None
+    _backend = ""
     _conn = None
 
     @abstractmethod
@@ -603,7 +603,7 @@ def close(self, **kwargs):
         if self._conn:
             self._conn.close()
 
-    @ait.deprecated(
+    @ait.deprecated(  # type: ignore
         "create_packets_from_results has been deprecated. Near equivalent functionality "
         "is available in create_packet_from_result."
     )

ait/core/dtype.py

@@ -71,6 +71,8 @@
 import sys
 import re
 
+from typing import Dict, Any
+
 from ait.core import cmd, dmc, log, util
 
 
@@ -90,7 +92,7 @@
 # ait.core.dtype.get(typename).  (Populated below based on
 # information in PrimitiveTypeFormats).
 #
-PrimitiveTypeMap = {}
+PrimitiveTypeMap: Dict[str, Any] = {}
 
 
 # PrimitiveTypeFormats

ait/core/evr.py

@@ -65,8 +65,8 @@ def load(self, content):
             log.error(msg)
             return
 
-        for e in evrs:
-            self.add(e)
+        for evr in evrs:
+            self.add(evr)
 
     def toJSON(self):
         return {code: defn.toJSON() for code, defn in self.items()}

ait/core/json.py

@@ -21,6 +21,7 @@
 
 import collections
 import json
+from typing import *  # type: ignore
 
 
 def slotsToJSON(obj, slots=None):
@@ -88,7 +89,7 @@ def toJSON(obj):
 
 
 class SlotSerializer(object):
-    __slots__ = []
+    __slots__: List[str] = []
 
     def __jsonOmit__(self, key, val):
         return val is None or val is ""

ait/core/server/broker.py

@@ -4,6 +4,8 @@
 
 gevent.monkey.patch_all()
 
+from typing import List, Any
+
 import ait.core
 import ait.core.server
 from ait.core import log
@@ -16,10 +18,10 @@ class Broker(gevent.Greenlet):
     This broker subscribes all ZMQ clients to their input topics.
     """
 
-    inbound_streams = []
-    outbound_streams = []
-    servers = []
-    plugins = []
+    inbound_streams: List[Any] = []
+    outbound_streams: List[Any] = []
+    servers: List[Any] = []
+    plugins: List[Any] = []
 
     def __init__(self):
         self.context = zmq.Context()

ait/core/server/test/test_handler.py

@@ -1,10 +1,10 @@
-import unittest
 import pickle
+import unittest
 from unittest import mock
 
 from ait.core import tlm
-from ait.core.server.handlers import PacketHandler
 from ait.core.server.handlers import CCSDSPacketHandler
+from ait.core.server.handlers import PacketHandler
 
 
 class TestCCSDSPacketCheck(unittest.TestCase):

ait/core/server/test/test_server.py

@@ -1,10 +1,9 @@
-import os
 import os.path
+from unittest import mock
+from unittest import TestCase
 
-from unittest import mock, TestCase
 import pytest
 
-import ait.core
 import ait.core.server
 from ait.core import cfg
 from ait.core.server.handlers import *