Merge branch 'master' into update_pip_dependencies

.github/workflows/build_ci.yml

@@ -0,0 +1,55 @@
+name: Build CI
+run-name: Build CI
+on:
+  pull_request:
+    branches: [ master ]
+  schedule:
+    - cron: "0 3 * * *"
+  workflow_dispatch:
+
+jobs:
+  build-ci:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ focal, jammy, buster, bullseye ]
+    runs-on: [ self-hosted, linux, x64, "${{ matrix.os }}" ]
+    timeout-minutes: 45
+    steps:
+      - name: Add Masks
+        run: |
+          echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}"
+          echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}"
+          echo "::add-mask::${{ secrets.CODECOV_TOKEN }}"
+          echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}"
+          echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}"
+          echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}"
+          echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}"
+          echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}"
+          echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}"
+          echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}"
+          echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}"
+          echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}"
+          echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}"
+          echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}"
+          echo "::add-mask::${{ secrets.SECRET_STRING_1 }}"
+          echo "::add-mask::${{ secrets.SECRET_STRING_2 }}"
+          echo "::add-mask::${{ secrets.SECRET_STRING_3 }}"
+      - name: Checkout Branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+      - name: Set ulimit
+        run: ulimit -n 9999
+      - name: Pre-Installation
+        shell: 'script -q -e -c "bash {0}"'
+        run: ./src/install/pre_install.sh
+      - name: Install FACT
+        shell: 'script -q -e -c "bash {0}"'
+        run: |
+          ./src/install.py -U -R -N -L DEBUG
+      - name: Unit Tests
+        shell: 'script -q -e -c "bash {0}"'
+        run: |
+          python3 -m pip install codecov
+          pytest --cov=.

src/analysis/PluginBase.py

@@ -39,10 +39,8 @@ class AnalysisBasePlugin(BasePlugin):  # pylint: disable=too-many-instance-attri
     MIME_BLACKLIST = []
     MIME_WHITELIST = []
 
-    def __init__(
-        self, plugin_administrator, config=None, no_multithread=False, offline_testing=False, view_updater=None
-    ):
-        super().__init__(plugin_administrator, config=config, plugin_path=self.FILE, view_updater=view_updater)
+    def __init__(self, config=None, no_multithread=False, offline_testing=False, view_updater=None):
+        super().__init__(config=config, plugin_path=self.FILE, view_updater=view_updater)
         self._check_plugin_attributes()
         self.check_config(no_multithread)
         self.additional_setup()
@@ -52,15 +50,13 @@ def __init__(
         self.workers = []
         self.thread_count = int(self.config[self.NAME]['threads'])
         self.active = [Value('i', 0) for _ in range(self.thread_count)]
-        self.register_plugin()
         if not offline_testing:
             self.start_worker()
 
     def additional_setup(self):
         '''
         This function can be implemented by the plugin to do initialization
         '''
-        pass
 
     def _check_plugin_attributes(self):
         for attribute in ['FILE', 'NAME', 'VERSION']:

src/analysis/YaraPluginBase.py

@@ -19,7 +19,7 @@ class YaraBasePlugin(AnalysisBasePlugin):
     VERSION = '0.0'
     FILE = None
 
-    def __init__(self, plugin_administrator, config=None, view_updater=None):
+    def __init__(self, config=None, view_updater=None):
         '''
         recursive flag: If True recursively analyze included files
         propagate flag: If True add analysis result of child to parent object
@@ -30,7 +30,7 @@ def __init__(self, plugin_administrator, config=None, view_updater=None):
             logging.error(f'Signature file {self.signature_path} not found. Did you run "compile_yara_signatures.py"?')
             raise PluginInitException(plugin=self)
         self.SYSTEM_VERSION = self.get_yara_system_version()  # pylint: disable=invalid-name
-        super().__init__(plugin_administrator, config=config, view_updater=view_updater)
+        super().__init__(config=config, view_updater=view_updater)
 
     def get_yara_system_version(self):
         with subprocess.Popen(['yara', '--version'], stdout=subprocess.PIPE) as process:

src/compare/PluginBase.py

@@ -13,10 +13,9 @@ class CompareBasePlugin(BasePlugin):
     # must be set by the plugin:
     FILE = None
 
-    def __init__(self, plugin_administrator, config=None, db_interface=None, view_updater=None):
-        super().__init__(plugin_administrator, config=config, plugin_path=self.FILE, view_updater=view_updater)
+    def __init__(self, config=None, db_interface=None, view_updater=None):
+        super().__init__(config=config, plugin_path=self.FILE, view_updater=view_updater)
         self.database = db_interface
-        self.register_plugin()
 
     @abstractmethod
     def compare_function(self, fo_list):

src/compare/compare.py

@@ -26,12 +26,12 @@ def __init__(self, config=None, db_interface: Optional[ComparisonDbInterface] =
 
     def compare(self, uid_list):
         logging.info(f'Compare in progress: {uid_list}')
-        bs = BinaryService(config=self.config)
+        binary_service = BinaryService(config=self.config)
 
         fo_list = []
         for uid in uid_list:
             fo = self.db_interface.get_complete_object_including_all_summaries(uid)
-            fo.binary = bs.get_binary_and_file_name(fo.uid)[0]
+            fo.binary = binary_service.get_binary_and_file_name(fo.uid)[0]
             fo_list.append(fo)
 
         return self.compare_objects(fo_list)
@@ -87,10 +87,9 @@ def _init_plugins(self):
                 # For why this exception can occur see Analysis.AnalysisScheduler.load_plugins
                 logging.error(f'Could not import plugin {plugin_name} due to exception', exc_info=True)
             else:
-                plugin.ComparePlugin(self, config=self.config, db_interface=self.db_interface)
-
-    def register_plugin(self, name, compare_plugin_instance):
-        self.compare_plugins[name] = compare_plugin_instance
+                self.compare_plugins[plugin.ComparePlugin.NAME] = plugin.ComparePlugin(
+                    config=self.config, db_interface=self.db_interface
+                )
 
     def _execute_compare_plugins(self, fo_list):
         return {name: plugin.compare(fo_list) for name, plugin in self.compare_plugins.items()}

src/install/pre_install.sh

@@ -73,7 +73,12 @@ sudo usermod -aG docker "$FACTUSER"
 
 # Setup npm repository as described in https://github.com/nodesource/distributions/blob/master/README.md#debinstall
 # This is required because the npm version that ships with Ubuntu 18.04 (bionic) is too old.
-curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash -
+if [ "${CODENAME}" = "bionic" ]; then
+  # the latest LTS release is too new for bionic (requires glibc 2.28), the next most recent one is 14
+  curl -fsSL https://deb.nodesource.com/setup_14.x | sudo -E bash -
+else
+  curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash -
+fi
 
 IS_VENV=$(python3 -c 'import sys; print(sys.exec_prefix!=sys.base_prefix)')
 SUDO=""

src/plugins/analysis/architecture_detection/code/architecture_detection.py

@@ -42,10 +42,10 @@ class AnalysisPlugin(AnalysisBasePlugin):
         'video',
     ]
 
-    def __init__(self, plugin_administrator, config=None):
+    def __init__(self, config=None):
         self.config = config
         self._fs_organizer = FSOrganizer(config)
-        super().__init__(plugin_administrator, config=config)
+        super().__init__(config=config)
 
     def process_object(self, file_object):
         arch_dict = construct_result(file_object, self._fs_organizer)

src/plugins/analysis/cve_lookup/test/test_cve_lookup.py

@@ -212,11 +212,6 @@ def test_search_cve_summary(monkeypatch):
         assert MATCHED_SUMMARY == actual_match
 
 
-class MockAdmin:
-    def register_plugin(self, name, administrator):
-        pass
-
-
 @pytest.fixture(scope='function')
 def test_config():
     return get_config_for_testing()
@@ -225,7 +220,7 @@ def test_config():
 @pytest.fixture(scope='function')
 def stub_plugin(test_config, monkeypatch):
     monkeypatch.setattr('plugins.base.BasePlugin._sync_view', lambda self, plugin_path: None)
-    return lookup.AnalysisPlugin(MockAdmin(), test_config, offline_testing=True)
+    return lookup.AnalysisPlugin(test_config, offline_testing=True)
 
 
 def test_process_object(stub_plugin):

src/plugins/analysis/elf_analysis/test/test_plugin_elf_analysis.py

@@ -15,11 +15,6 @@
 TEST_DATA_DIR = Path(__file__).parent / 'data'
 
 
-class MockAdmin:
-    def register_plugin(self, name, administrator):
-        pass
-
-
 LiefResult = namedtuple(
     'LiefResult', ['symbols_version', 'libraries', 'imported_functions', 'exported_functions', 'sections']
 )
@@ -57,7 +52,7 @@ def stub_object():
 @pytest.fixture(scope='function')
 def stub_plugin(test_config, monkeypatch):
     monkeypatch.setattr('plugins.base.BasePlugin._sync_view', lambda self, plugin_path: None)
-    return AnalysisPlugin(MockAdmin(), test_config, offline_testing=True)
+    return AnalysisPlugin(test_config, offline_testing=True)
 
 
 @pytest.mark.parametrize(

src/plugins/analysis/file_system_metadata/test/test_plugin_file_system_metadata.py

@@ -64,7 +64,7 @@ def setUp(self):
         self.test_file_fs = TEST_DATA_DIR / 'squashfs.img'
 
     def setup_plugin(self):
-        return AnalysisPlugin(self, config=self.config, view_updater=CommonDatabaseMock(), db_interface=DbMock())
+        return AnalysisPlugin(config=self.config, view_updater=CommonDatabaseMock(), db_interface=DbMock())
 
     def _extract_metadata_from_archive_mock(self, _):
         self.result = 'archive'

src/plugins/analysis/hashlookup/test/test_hashlookup.py

@@ -8,11 +8,6 @@
 KNOWN_ZSH_HASH = 'A6F2177402114FC8B5E7ECF924FFA61A2AC25BD347BC3370FB92E07B76E0B44C'
 
 
-class MockAdmin:
-    def register_plugin(self, name, administrator):
-        pass
-
-
 @pytest.fixture(scope='function')
 def test_config():
     return get_config_for_testing()
@@ -21,7 +16,7 @@ def test_config():
 @pytest.fixture(scope='function')
 def stub_plugin(test_config, monkeypatch):
     monkeypatch.setattr('plugins.base.BasePlugin._sync_view', lambda self, plugin_path: None)
-    return AnalysisPlugin(MockAdmin(), test_config, offline_testing=True)
+    return AnalysisPlugin(test_config, offline_testing=True)
 
 
 @pytest.fixture(scope='function')

src/plugins/analysis/linter/test/test_source_code_analysis.py

@@ -12,11 +12,6 @@
 PYLINT_TEST_FILE = Path(__file__).parent / 'data' / 'linter_test_file'
 
 
-class MockAdmin:
-    def register_plugin(self, name, administrator):
-        pass
-
-
 @pytest.fixture(scope='function')
 def test_config():
     return get_config_for_testing()
@@ -29,7 +24,7 @@ def test_object():
 
 @pytest.fixture(scope='function')
 def stub_plugin(test_config, monkeypatch):
-    return AnalysisPlugin(MockAdmin(), test_config, offline_testing=True, view_updater=CommonDatabaseMock())
+    return AnalysisPlugin(test_config, offline_testing=True, view_updater=CommonDatabaseMock())
 
 
 def test_process_object_not_supported(stub_plugin, test_object, monkeypatch):

src/plugins/analysis/qemu_exec/test/test_plugin_qemu_exec.py

@@ -96,7 +96,7 @@ class TestPluginQemuExec(AnalysisPluginTest):
     PLUGIN_CLASS = AnalysisPlugin
 
     def setup_plugin(self):
-        return AnalysisPlugin(self, config=self.config, unpacker=MockUnpacker(), view_updater=CommonDatabaseMock())
+        return AnalysisPlugin(config=self.config, unpacker=MockUnpacker(), view_updater=CommonDatabaseMock())
 
     def test_has_relevant_type(self):
         assert self.analysis_plugin._has_relevant_type(None) is False

src/plugins/analysis/tlsh/test/test_plugin_tlsh.py

@@ -9,11 +9,6 @@
 HASH_1 = '0CC34B06B1B258BCC16689308A67D671AB747E5053223B3E3684F7342F56E6F1F0DAB1'
 
 
-class MockAdmin:
-    def register_plugin(self, name, administrator):
-        pass
-
-
 class MockDb:
     def get_all_tlsh_hashes(self):  # pylint: disable=no-self-use
         return [('test_uid', HASH_1)]
@@ -34,7 +29,7 @@ def test_object():
 @pytest.fixture(scope='function')
 def stub_plugin(test_config):
     return AnalysisPlugin(
-        MockAdmin(), config=test_config, offline_testing=True, view_updater=CommonDatabaseMock(), db_interface=MockDb()
+        config=test_config, offline_testing=True, view_updater=CommonDatabaseMock(), db_interface=MockDb()
     )
 
 

src/plugins/base.py

@@ -9,8 +9,7 @@ class BasePlugin:
     NAME = 'base'
     DEPENDENCIES = []
 
-    def __init__(self, plugin_administrator, config=None, plugin_path=None, view_updater=None):
-        self.plugin_administrator = plugin_administrator
+    def __init__(self, config=None, plugin_path=None, view_updater=None):
         self.config = config
         self.view_updater = view_updater if view_updater is not None else ViewUpdater(config)
         if plugin_path:
@@ -32,6 +31,3 @@ def _get_view_file_path(cls, plugin_path: str) -> Optional[Path]:
         if len(view_files) > 1:
             logging.warning(f'{cls.NAME}: Plug-in provides more than one view! \'{view_files[0]}\' is used!')
         return view_files[0]
-
-    def register_plugin(self):
-        self.plugin_administrator.register_plugin(self.NAME, self)

src/plugins/compare/file_coverage/test/test_plugin_file_coverage.py

@@ -25,7 +25,7 @@ def setup_plugin(self):
         This function must be overwritten by the test instance.
         In most cases it is sufficient to copy this function.
         '''
-        return ComparePlugin(self, config=self.config, db_interface=DbMock(), view_updater=CommonDatabaseMock())
+        return ComparePlugin(config=self.config, db_interface=DbMock(), view_updater=CommonDatabaseMock())
 
     def test_get_intersection_of_files(self):
         self.fw_one.list_of_all_included_files.append('foo')

src/scheduler/analysis.py

@@ -184,18 +184,7 @@ def _load_plugins(self):
                 # be missing dependencies. So if anything goes wrong we want to inform the user about it
                 logging.error(f'Could not import plugin {plugin_name} due to exception', exc_info=True)
             else:
-                plugin.AnalysisPlugin(self, config=self.config)
-
-    def register_plugin(self, name: str, plugin_instance: AnalysisBasePlugin):
-        '''
-        This function is used by analysis plugins to register themselves with this scheduler. During initialization the
-        plugins will call this functions giving their name and a reference to their object to allow the scheduler to
-        address them for running analyses.
-
-        :param name: The plugin name for addressing in runner and collector
-        :param plugin_instance: A reference to the plugin object
-        '''
-        self.analysis_plugins[name] = plugin_instance
+                self.analysis_plugins[plugin.AnalysisPlugin.NAME] = plugin.AnalysisPlugin(config=self.config)
 
     def _get_plugin_sets_from_config(self):
         try:

src/test/unit/analysis/analysis_plugin_test_class.py

@@ -1,5 +1,6 @@
 import unittest.mock
 from configparser import ConfigParser
+from typing import Callable
 
 from test.common_helper import (  # pylint: disable=wrong-import-order
     CommonDatabaseMock,
@@ -15,7 +16,7 @@ class AnalysisPluginTest(unittest.TestCase):
 
     # must be set by individual plugin test class
     PLUGIN_NAME = 'plugin_test'
-    PLUGIN_CLASS = None
+    PLUGIN_CLASS: Callable = None
 
     def setUp(self):
         self.docker_mount_base_dir = create_docker_mount_base_dir()
@@ -28,9 +29,7 @@ def _set_config(self):
 
     def setup_plugin(self):
         # overwrite in plugin tests if necessary
-        return self.PLUGIN_CLASS(
-            self, config=self.config, view_updater=CommonDatabaseMock()
-        )  # pylint: disable=not-callable
+        return self.PLUGIN_CLASS(config=self.config, view_updater=CommonDatabaseMock())
 
     def tearDown(self):
         self.analysis_plugin.shutdown()  # pylint: disable=no-member
@@ -50,13 +49,3 @@ def init_basic_config(self):
         config.set('data-storage', 'postgres-database', 'fact-test')
 
         return config
-
-    def register_plugin(self, name, plugin_object):
-        '''
-        This is a mock checking if the plugin registers correctly
-        '''
-        self.assertEqual(name, self.PLUGIN_NAME, 'plugin registers with wrong name')
-        self.assertEqual(plugin_object.NAME, self.PLUGIN_NAME, 'plugin object has wrong name')
-        self.assertIsInstance(plugin_object.DESCRIPTION, str)
-        self.assertIsInstance(plugin_object.VERSION, str)
-        self.assertNotEqual(plugin_object.VERSION, 'not set', 'Plug-in version not set')