From fbcf34ed43bfb555cf56f755610ffd47dc66f14d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Wed, 9 Nov 2022 17:01:32 +0100 Subject: [PATCH 01/16] added github action for ci --- .github/workflows/ci_focal.yml | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/ci_focal.yml diff --git a/.github/workflows/ci_focal.yml b/.github/workflows/ci_focal.yml new file mode 100644 index 0000000000..a25087cae8 --- /dev/null +++ b/.github/workflows/ci_focal.yml @@ -0,0 +1,46 @@ +name: CI/Focal +run-name: CI/Focal +on: + pull_request: + branches: [ master ] + workflow_dispatch: + +jobs: + build: + runs-on: [self-hosted, linux, x64, focal] + timeout-minutes: 45 + steps: + - name: Add Masks + run: | + echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" + echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" + echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" + - name: Checkout Branch + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + - name: Set ulimit + run: ulimit -n 9999 + - name: Pre-Installation + shell: 'script -q -e -c "bash {0}"' + run: ./src/install/pre_install.sh + - name: Install FACT + shell: 'script -q -e -c "bash {0}"' + run: | + ./src/install.py -U -R -N -L DEBUG + - name: Unit Tests + shell: 'script -q -e -c "bash {0}"' + run: | + python3 -m pip install codecov + pytest --cov=. From 9d1729bedc1156cc81e43881bc8250a508d464be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Thu, 10 Nov 2022 09:28:41 +0100 Subject: [PATCH 02/16] added more masks --- .github/workflows/ci_focal.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci_focal.yml b/.github/workflows/ci_focal.yml index a25087cae8..6e5e92429b 100644 --- a/.github/workflows/ci_focal.yml +++ b/.github/workflows/ci_focal.yml @@ -26,6 +26,9 @@ jobs: echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" - name: Checkout Branch uses: actions/checkout@v3 with: From dc6ff703aab6ee2a54a3606251235422bd92c5d0 Mon Sep 17 00:00:00 2001 From: rhelmke Date: Thu, 10 Nov 2022 10:39:24 +0100 Subject: [PATCH 03/16] add jammy worker --- .github/workflows/build_ci.yml | 132 +++++++++++++++++++++++++++++++++ .github/workflows/ci_focal.yml | 49 ------------ 2 files changed, 132 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/build_ci.yml delete mode 100644 .github/workflows/ci_focal.yml diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml new file mode 100644 index 0000000000..7d8636a01f --- /dev/null +++ b/.github/workflows/build_ci.yml @@ -0,0 +1,132 @@ +name: Build CI +run-name: Build CI +on: + pull_request: + branches: [ master ] + workflow_dispatch: + +jobs: + focal: + runs-on: [self-hosted, linux, x64, focal] + timeout-minutes: 45 + steps: + - name: Add Masks + run: | + echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" + echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" + echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" + - name: Checkout Branch + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + - name: Set ulimit + run: ulimit -n 9999 + - name: Pre-Installation + shell: 'script -q -e -c "bash {0}"' + run: ./src/install/pre_install.sh + - name: Install FACT + shell: 'script -q -e -c "bash {0}"' + run: | + ./src/install.py -U -R -N -L DEBUG + - name: Unit Tests + shell: 'script -q -e -c "bash {0}"' + run: | + python3 -m pip install codecov + pytest --cov=. +jobs: + focal: + runs-on: [self-hosted, linux, x64, focal] + timeout-minutes: 45 + steps: + - name: Add Masks + run: | + echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" + echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" + echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" + - name: Checkout Branch + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + - name: Set ulimit + run: ulimit -n 9999 + - name: Pre-Installation + shell: 'script -q -e -c "bash {0}"' + run: ./src/install/pre_install.sh + - name: Install FACT + shell: 'script -q -e -c "bash {0}"' + run: | + ./src/install.py -U -R -N -L DEBUG + - name: Unit Tests + shell: 'script -q -e -c "bash {0}"' + run: | + python3 -m pip install codecov + pytest --cov=. + jammy: + runs-on: [self-hosted, linux, x64, jammy] + timeout-minutes: 45 + steps: + - name: Add Masks + run: | + echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" + echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" + echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" + - name: Checkout Branch + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + - name: Set ulimit + run: ulimit -n 9999 + - name: Pre-Installation + shell: 'script -q -e -c "bash {0}"' + run: ./src/install/pre_install.sh + - name: Install FACT + shell: 'script -q -e -c "bash {0}"' + run: | + ./src/install.py -U -R -N -L DEBUG + - name: Unit Tests + shell: 'script -q -e -c "bash {0}"' + run: | + python3 -m pip install codecov + pytest --cov=. diff --git a/.github/workflows/ci_focal.yml b/.github/workflows/ci_focal.yml deleted file mode 100644 index 6e5e92429b..0000000000 --- a/.github/workflows/ci_focal.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: CI/Focal -run-name: CI/Focal -on: - pull_request: - branches: [ master ] - workflow_dispatch: - -jobs: - build: - runs-on: [self-hosted, linux, x64, focal] - timeout-minutes: 45 - steps: - - name: Add Masks - run: | - echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" - echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" - echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" - - name: Checkout Branch - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - name: Set ulimit - run: ulimit -n 9999 - - name: Pre-Installation - shell: 'script -q -e -c "bash {0}"' - run: ./src/install/pre_install.sh - - name: Install FACT - shell: 'script -q -e -c "bash {0}"' - run: | - ./src/install.py -U -R -N -L DEBUG - - name: Unit Tests - shell: 'script -q -e -c "bash {0}"' - run: | - python3 -m pip install codecov - pytest --cov=. From ba41f9662977db7346e8a2bfe33c5e674e01ce60 Mon Sep 17 00:00:00 2001 From: rhelmke Date: Thu, 10 Nov 2022 10:41:00 +0100 Subject: [PATCH 04/16] copy-paste error fix for jammy --- .github/workflows/build_ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml index 7d8636a01f..84622ff24b 100644 --- a/.github/workflows/build_ci.yml +++ b/.github/workflows/build_ci.yml @@ -47,9 +47,8 @@ jobs: run: | python3 -m pip install codecov pytest --cov=. -jobs: - focal: - runs-on: [self-hosted, linux, x64, focal] + jammy: + runs-on: [self-hosted, linux, x64, jammy] timeout-minutes: 45 steps: - name: Add Masks From 05697bfa656850ec4b4bc2b707676efdcd8442fd Mon Sep 17 00:00:00 2001 From: rhelmke Date: Thu, 10 Nov 2022 10:42:53 +0100 Subject: [PATCH 05/16] more copy-paste fixes... --- .github/workflows/build_ci.yml | 41 ---------------------------------- 1 file changed, 41 deletions(-) diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml index 84622ff24b..777fdccd01 100644 --- a/.github/workflows/build_ci.yml +++ b/.github/workflows/build_ci.yml @@ -88,44 +88,3 @@ jobs: run: | python3 -m pip install codecov pytest --cov=. - jammy: - runs-on: [self-hosted, linux, x64, jammy] - timeout-minutes: 45 - steps: - - name: Add Masks - run: | - echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" - echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" - echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" - - name: Checkout Branch - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - name: Set ulimit - run: ulimit -n 9999 - - name: Pre-Installation - shell: 'script -q -e -c "bash {0}"' - run: ./src/install/pre_install.sh - - name: Install FACT - shell: 'script -q -e -c "bash {0}"' - run: | - ./src/install.py -U -R -N -L DEBUG - - name: Unit Tests - shell: 'script -q -e -c "bash {0}"' - run: | - python3 -m pip install codecov - pytest --cov=. From 46df7381fa10e224a406991ee4eba9ab7a6ac183 Mon Sep 17 00:00:00 2001 From: rhelmke Date: Thu, 10 Nov 2022 12:30:18 +0100 Subject: [PATCH 06/16] add bullseye and buster jobs --- .github/workflows/build_ci.yml | 85 ++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml index 777fdccd01..8a70914ff5 100644 --- a/.github/workflows/build_ci.yml +++ b/.github/workflows/build_ci.yml @@ -47,6 +47,7 @@ jobs: run: | python3 -m pip install codecov pytest --cov=. + jammy: runs-on: [self-hosted, linux, x64, jammy] timeout-minutes: 45 @@ -88,3 +89,87 @@ jobs: run: | python3 -m pip install codecov pytest --cov=. + + buster: + runs-on: [self-hosted, linux, x64, buster] + timeout-minutes: 45 + steps: + - name: Add Masks + run: | + echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" + echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" + echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" + - name: Checkout Branch + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + - name: Set ulimit + run: ulimit -n 9999 - name: Pre-Installation + shell: 'script -q -e -c "bash {0}"' + run: ./src/install/pre_install.sh + - name: Install FACT + shell: 'script -q -e -c "bash {0}"' + run: | + ./src/install.py -U -R -N -L DEBUG + - name: Unit Tests + shell: 'script -q -e -c "bash {0}"' + run: | + python3 -m pip install codecov + pytest --cov=. + + bullseye: + runs-on: [self-hosted, linux, x64, bullseye] + timeout-minutes: 45 + steps: + - name: Add Masks + run: | + echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" + echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" + echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" + echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" + echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" + echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" + - name: Checkout Branch + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + - name: Set ulimit + run: ulimit -n 9999 + - name: Pre-Installation + shell: 'script -q -e -c "bash {0}"' + run: ./src/install/pre_install.sh + - name: Install FACT + shell: 'script -q -e -c "bash {0}"' + run: | + ./src/install.py -U -R -N -L DEBUG + - name: Unit Tests + shell: 'script -q -e -c "bash {0}"' + run: | + python3 -m pip install codecov + pytest --cov=. + From 9e4ce551b15c88dcd4675ec87b89daeac0b8bd56 Mon Sep 17 00:00:00 2001 From: rhelmke Date: Thu, 10 Nov 2022 13:06:15 +0100 Subject: [PATCH 07/16] added newline to ci action --- .github/workflows/build_ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml index 8a70914ff5..095fcb962e 100644 --- a/.github/workflows/build_ci.yml +++ b/.github/workflows/build_ci.yml @@ -118,7 +118,8 @@ jobs: with: ref: ${{ github.ref }} - name: Set ulimit - run: ulimit -n 9999 - name: Pre-Installation + run: ulimit -n 9999 + - name: Pre-Installation shell: 'script -q -e -c "bash {0}"' run: ./src/install/pre_install.sh - name: Install FACT From 4c4d12bd81ddcef944c3ddbc46dcc50e2339745e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Fri, 21 Oct 2022 14:54:10 +0200 Subject: [PATCH 08/16] removed `register_plugin` construct --- src/analysis/PluginBase.py | 2 -- src/compare/PluginBase.py | 1 - src/compare/compare.py | 11 +++++------ src/plugins/base.py | 3 --- src/scheduler/analysis.py | 13 +------------ 5 files changed, 6 insertions(+), 24 deletions(-) diff --git a/src/analysis/PluginBase.py b/src/analysis/PluginBase.py index bf6fdf655b..c61af27222 100644 --- a/src/analysis/PluginBase.py +++ b/src/analysis/PluginBase.py @@ -52,7 +52,6 @@ def __init__( self.workers = [] self.thread_count = int(self.config[self.NAME]['threads']) self.active = [Value('i', 0) for _ in range(self.thread_count)] - self.register_plugin() if not offline_testing: self.start_worker() @@ -60,7 +59,6 @@ def additional_setup(self): ''' This function can be implemented by the plugin to do initialization ''' - pass def _check_plugin_attributes(self): for attribute in ['FILE', 'NAME', 'VERSION']: diff --git a/src/compare/PluginBase.py b/src/compare/PluginBase.py index 9ca662ccdf..d3ead78611 100644 --- a/src/compare/PluginBase.py +++ b/src/compare/PluginBase.py @@ -16,7 +16,6 @@ class CompareBasePlugin(BasePlugin): def __init__(self, plugin_administrator, config=None, db_interface=None, view_updater=None): super().__init__(plugin_administrator, config=config, plugin_path=self.FILE, view_updater=view_updater) self.database = db_interface - self.register_plugin() @abstractmethod def compare_function(self, fo_list): diff --git a/src/compare/compare.py b/src/compare/compare.py index 1c172049f5..8c01795efa 100644 --- a/src/compare/compare.py +++ b/src/compare/compare.py @@ -26,12 +26,12 @@ def __init__(self, config=None, db_interface: Optional[ComparisonDbInterface] = def compare(self, uid_list): logging.info(f'Compare in progress: {uid_list}') - bs = BinaryService(config=self.config) + binary_service = BinaryService(config=self.config) fo_list = [] for uid in uid_list: fo = self.db_interface.get_complete_object_including_all_summaries(uid) - fo.binary = bs.get_binary_and_file_name(fo.uid)[0] + fo.binary = binary_service.get_binary_and_file_name(fo.uid)[0] fo_list.append(fo) return self.compare_objects(fo_list) @@ -87,10 +87,9 @@ def _init_plugins(self): # For why this exception can occur see Analysis.AnalysisScheduler.load_plugins logging.error(f'Could not import plugin {plugin_name} due to exception', exc_info=True) else: - plugin.ComparePlugin(self, config=self.config, db_interface=self.db_interface) - - def register_plugin(self, name, compare_plugin_instance): - self.compare_plugins[name] = compare_plugin_instance + self.compare_plugins[plugin.ComparePlugin.NAME] = plugin.ComparePlugin( + self, config=self.config, db_interface=self.db_interface + ) def _execute_compare_plugins(self, fo_list): return {name: plugin.compare(fo_list) for name, plugin in self.compare_plugins.items()} diff --git a/src/plugins/base.py b/src/plugins/base.py index 290e3b2aa0..e378aff53d 100644 --- a/src/plugins/base.py +++ b/src/plugins/base.py @@ -32,6 +32,3 @@ def _get_view_file_path(cls, plugin_path: str) -> Optional[Path]: if len(view_files) > 1: logging.warning(f'{cls.NAME}: Plug-in provides more than one view! \'{view_files[0]}\' is used!') return view_files[0] - - def register_plugin(self): - self.plugin_administrator.register_plugin(self.NAME, self) diff --git a/src/scheduler/analysis.py b/src/scheduler/analysis.py index 276ea9b026..415d428729 100644 --- a/src/scheduler/analysis.py +++ b/src/scheduler/analysis.py @@ -184,18 +184,7 @@ def _load_plugins(self): # be missing dependencies. So if anything goes wrong we want to inform the user about it logging.error(f'Could not import plugin {plugin_name} due to exception', exc_info=True) else: - plugin.AnalysisPlugin(self, config=self.config) - - def register_plugin(self, name: str, plugin_instance: AnalysisBasePlugin): - ''' - This function is used by analysis plugins to register themselves with this scheduler. During initialization the - plugins will call this functions giving their name and a reference to their object to allow the scheduler to - address them for running analyses. - - :param name: The plugin name for addressing in runner and collector - :param plugin_instance: A reference to the plugin object - ''' - self.analysis_plugins[name] = plugin_instance + self.analysis_plugins[plugin.AnalysisPlugin.NAME] = plugin.AnalysisPlugin(self, config=self.config) def _get_plugin_sets_from_config(self): try: From 2e8e98a5dd55dec57055a095ea047b8b0c4556a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Fri, 21 Oct 2022 15:27:05 +0200 Subject: [PATCH 09/16] removed `plugin_administrator` construct --- src/analysis/PluginBase.py | 6 ++---- src/analysis/YaraPluginBase.py | 4 ++-- src/compare/PluginBase.py | 4 ++-- src/compare/compare.py | 2 +- .../code/architecture_detection.py | 4 ++-- .../cve_lookup/test/test_cve_lookup.py | 7 +------ .../test/test_plugin_elf_analysis.py | 7 +------ .../test/test_plugin_file_system_metadata.py | 2 +- .../hashlookup/test/test_hashlookup.py | 7 +------ .../linter/test/test_source_code_analysis.py | 7 +------ .../qemu_exec/test/test_plugin_qemu_exec.py | 2 +- .../analysis/tlsh/test/test_plugin_tlsh.py | 7 +------ src/plugins/base.py | 3 +-- .../test/test_plugin_file_coverage.py | 2 +- src/scheduler/analysis.py | 2 +- .../analysis/analysis_plugin_test_class.py | 17 +++------------- src/test/unit/analysis/test_plugin_base.py | 20 +++++-------------- .../unit/compare/compare_plugin_test_class.py | 12 +---------- src/test/unit/compare/test_plugin_base.py | 2 +- 19 files changed, 29 insertions(+), 88 deletions(-) diff --git a/src/analysis/PluginBase.py b/src/analysis/PluginBase.py index c61af27222..17dafec02e 100644 --- a/src/analysis/PluginBase.py +++ b/src/analysis/PluginBase.py @@ -39,10 +39,8 @@ class AnalysisBasePlugin(BasePlugin): # pylint: disable=too-many-instance-attri MIME_BLACKLIST = [] MIME_WHITELIST = [] - def __init__( - self, plugin_administrator, config=None, no_multithread=False, offline_testing=False, view_updater=None - ): - super().__init__(plugin_administrator, config=config, plugin_path=self.FILE, view_updater=view_updater) + def __init__(self, config=None, no_multithread=False, offline_testing=False, view_updater=None): + super().__init__(config=config, plugin_path=self.FILE, view_updater=view_updater) self._check_plugin_attributes() self.check_config(no_multithread) self.additional_setup() diff --git a/src/analysis/YaraPluginBase.py b/src/analysis/YaraPluginBase.py index 16ed889e37..61d9498b6b 100644 --- a/src/analysis/YaraPluginBase.py +++ b/src/analysis/YaraPluginBase.py @@ -19,7 +19,7 @@ class YaraBasePlugin(AnalysisBasePlugin): VERSION = '0.0' FILE = None - def __init__(self, plugin_administrator, config=None, view_updater=None): + def __init__(self, config=None, view_updater=None): ''' recursive flag: If True recursively analyze included files propagate flag: If True add analysis result of child to parent object @@ -30,7 +30,7 @@ def __init__(self, plugin_administrator, config=None, view_updater=None): logging.error(f'Signature file {self.signature_path} not found. Did you run "compile_yara_signatures.py"?') raise PluginInitException(plugin=self) self.SYSTEM_VERSION = self.get_yara_system_version() # pylint: disable=invalid-name - super().__init__(plugin_administrator, config=config, view_updater=view_updater) + super().__init__(config=config, view_updater=view_updater) def get_yara_system_version(self): with subprocess.Popen(['yara', '--version'], stdout=subprocess.PIPE) as process: diff --git a/src/compare/PluginBase.py b/src/compare/PluginBase.py index d3ead78611..a94faa9d70 100644 --- a/src/compare/PluginBase.py +++ b/src/compare/PluginBase.py @@ -13,8 +13,8 @@ class CompareBasePlugin(BasePlugin): # must be set by the plugin: FILE = None - def __init__(self, plugin_administrator, config=None, db_interface=None, view_updater=None): - super().__init__(plugin_administrator, config=config, plugin_path=self.FILE, view_updater=view_updater) + def __init__(self, config=None, db_interface=None, view_updater=None): + super().__init__(config=config, plugin_path=self.FILE, view_updater=view_updater) self.database = db_interface @abstractmethod diff --git a/src/compare/compare.py b/src/compare/compare.py index 8c01795efa..894e035ef1 100644 --- a/src/compare/compare.py +++ b/src/compare/compare.py @@ -88,7 +88,7 @@ def _init_plugins(self): logging.error(f'Could not import plugin {plugin_name} due to exception', exc_info=True) else: self.compare_plugins[plugin.ComparePlugin.NAME] = plugin.ComparePlugin( - self, config=self.config, db_interface=self.db_interface + config=self.config, db_interface=self.db_interface ) def _execute_compare_plugins(self, fo_list): diff --git a/src/plugins/analysis/architecture_detection/code/architecture_detection.py b/src/plugins/analysis/architecture_detection/code/architecture_detection.py index 4e785afd2d..34bc49c4a5 100644 --- a/src/plugins/analysis/architecture_detection/code/architecture_detection.py +++ b/src/plugins/analysis/architecture_detection/code/architecture_detection.py @@ -42,10 +42,10 @@ class AnalysisPlugin(AnalysisBasePlugin): 'video', ] - def __init__(self, plugin_administrator, config=None): + def __init__(self, config=None): self.config = config self._fs_organizer = FSOrganizer(config) - super().__init__(plugin_administrator, config=config) + super().__init__(config=config) def process_object(self, file_object): arch_dict = construct_result(file_object, self._fs_organizer) diff --git a/src/plugins/analysis/cve_lookup/test/test_cve_lookup.py b/src/plugins/analysis/cve_lookup/test/test_cve_lookup.py index 55db8349dd..79b12054cc 100644 --- a/src/plugins/analysis/cve_lookup/test/test_cve_lookup.py +++ b/src/plugins/analysis/cve_lookup/test/test_cve_lookup.py @@ -212,11 +212,6 @@ def test_search_cve_summary(monkeypatch): assert MATCHED_SUMMARY == actual_match -class MockAdmin: - def register_plugin(self, name, administrator): - pass - - @pytest.fixture(scope='function') def test_config(): return get_config_for_testing() @@ -225,7 +220,7 @@ def test_config(): @pytest.fixture(scope='function') def stub_plugin(test_config, monkeypatch): monkeypatch.setattr('plugins.base.BasePlugin._sync_view', lambda self, plugin_path: None) - return lookup.AnalysisPlugin(MockAdmin(), test_config, offline_testing=True) + return lookup.AnalysisPlugin(test_config, offline_testing=True) def test_process_object(stub_plugin): diff --git a/src/plugins/analysis/elf_analysis/test/test_plugin_elf_analysis.py b/src/plugins/analysis/elf_analysis/test/test_plugin_elf_analysis.py index 1fd68d4d45..e066049b65 100644 --- a/src/plugins/analysis/elf_analysis/test/test_plugin_elf_analysis.py +++ b/src/plugins/analysis/elf_analysis/test/test_plugin_elf_analysis.py @@ -15,11 +15,6 @@ TEST_DATA_DIR = Path(__file__).parent / 'data' -class MockAdmin: - def register_plugin(self, name, administrator): - pass - - LiefResult = namedtuple( 'LiefResult', ['symbols_version', 'libraries', 'imported_functions', 'exported_functions', 'sections'] ) @@ -57,7 +52,7 @@ def stub_object(): @pytest.fixture(scope='function') def stub_plugin(test_config, monkeypatch): monkeypatch.setattr('plugins.base.BasePlugin._sync_view', lambda self, plugin_path: None) - return AnalysisPlugin(MockAdmin(), test_config, offline_testing=True) + return AnalysisPlugin(test_config, offline_testing=True) @pytest.mark.parametrize( diff --git a/src/plugins/analysis/file_system_metadata/test/test_plugin_file_system_metadata.py b/src/plugins/analysis/file_system_metadata/test/test_plugin_file_system_metadata.py index 2c45d7a294..c44cf7a5fb 100644 --- a/src/plugins/analysis/file_system_metadata/test/test_plugin_file_system_metadata.py +++ b/src/plugins/analysis/file_system_metadata/test/test_plugin_file_system_metadata.py @@ -64,7 +64,7 @@ def setUp(self): self.test_file_fs = TEST_DATA_DIR / 'squashfs.img' def setup_plugin(self): - return AnalysisPlugin(self, config=self.config, view_updater=CommonDatabaseMock(), db_interface=DbMock()) + return AnalysisPlugin(config=self.config, view_updater=CommonDatabaseMock(), db_interface=DbMock()) def _extract_metadata_from_archive_mock(self, _): self.result = 'archive' diff --git a/src/plugins/analysis/hashlookup/test/test_hashlookup.py b/src/plugins/analysis/hashlookup/test/test_hashlookup.py index 068675ea90..c07fa97b16 100644 --- a/src/plugins/analysis/hashlookup/test/test_hashlookup.py +++ b/src/plugins/analysis/hashlookup/test/test_hashlookup.py @@ -8,11 +8,6 @@ KNOWN_ZSH_HASH = 'A6F2177402114FC8B5E7ECF924FFA61A2AC25BD347BC3370FB92E07B76E0B44C' -class MockAdmin: - def register_plugin(self, name, administrator): - pass - - @pytest.fixture(scope='function') def test_config(): return get_config_for_testing() @@ -21,7 +16,7 @@ def test_config(): @pytest.fixture(scope='function') def stub_plugin(test_config, monkeypatch): monkeypatch.setattr('plugins.base.BasePlugin._sync_view', lambda self, plugin_path: None) - return AnalysisPlugin(MockAdmin(), test_config, offline_testing=True) + return AnalysisPlugin(test_config, offline_testing=True) @pytest.fixture(scope='function') diff --git a/src/plugins/analysis/linter/test/test_source_code_analysis.py b/src/plugins/analysis/linter/test/test_source_code_analysis.py index 477de16450..da889207b6 100644 --- a/src/plugins/analysis/linter/test/test_source_code_analysis.py +++ b/src/plugins/analysis/linter/test/test_source_code_analysis.py @@ -12,11 +12,6 @@ PYLINT_TEST_FILE = Path(__file__).parent / 'data' / 'linter_test_file' -class MockAdmin: - def register_plugin(self, name, administrator): - pass - - @pytest.fixture(scope='function') def test_config(): return get_config_for_testing() @@ -29,7 +24,7 @@ def test_object(): @pytest.fixture(scope='function') def stub_plugin(test_config, monkeypatch): - return AnalysisPlugin(MockAdmin(), test_config, offline_testing=True, view_updater=CommonDatabaseMock()) + return AnalysisPlugin(test_config, offline_testing=True, view_updater=CommonDatabaseMock()) def test_process_object_not_supported(stub_plugin, test_object, monkeypatch): diff --git a/src/plugins/analysis/qemu_exec/test/test_plugin_qemu_exec.py b/src/plugins/analysis/qemu_exec/test/test_plugin_qemu_exec.py index a0349cdf32..483756135a 100644 --- a/src/plugins/analysis/qemu_exec/test/test_plugin_qemu_exec.py +++ b/src/plugins/analysis/qemu_exec/test/test_plugin_qemu_exec.py @@ -96,7 +96,7 @@ class TestPluginQemuExec(AnalysisPluginTest): PLUGIN_CLASS = AnalysisPlugin def setup_plugin(self): - return AnalysisPlugin(self, config=self.config, unpacker=MockUnpacker(), view_updater=CommonDatabaseMock()) + return AnalysisPlugin(config=self.config, unpacker=MockUnpacker(), view_updater=CommonDatabaseMock()) def test_has_relevant_type(self): assert self.analysis_plugin._has_relevant_type(None) is False diff --git a/src/plugins/analysis/tlsh/test/test_plugin_tlsh.py b/src/plugins/analysis/tlsh/test/test_plugin_tlsh.py index dc30cfa92c..e0770ae08b 100644 --- a/src/plugins/analysis/tlsh/test/test_plugin_tlsh.py +++ b/src/plugins/analysis/tlsh/test/test_plugin_tlsh.py @@ -9,11 +9,6 @@ HASH_1 = '0CC34B06B1B258BCC16689308A67D671AB747E5053223B3E3684F7342F56E6F1F0DAB1' -class MockAdmin: - def register_plugin(self, name, administrator): - pass - - class MockDb: def get_all_tlsh_hashes(self): # pylint: disable=no-self-use return [('test_uid', HASH_1)] @@ -34,7 +29,7 @@ def test_object(): @pytest.fixture(scope='function') def stub_plugin(test_config): return AnalysisPlugin( - MockAdmin(), config=test_config, offline_testing=True, view_updater=CommonDatabaseMock(), db_interface=MockDb() + config=test_config, offline_testing=True, view_updater=CommonDatabaseMock(), db_interface=MockDb() ) diff --git a/src/plugins/base.py b/src/plugins/base.py index e378aff53d..4a2c2a37a8 100644 --- a/src/plugins/base.py +++ b/src/plugins/base.py @@ -9,8 +9,7 @@ class BasePlugin: NAME = 'base' DEPENDENCIES = [] - def __init__(self, plugin_administrator, config=None, plugin_path=None, view_updater=None): - self.plugin_administrator = plugin_administrator + def __init__(self, config=None, plugin_path=None, view_updater=None): self.config = config self.view_updater = view_updater if view_updater is not None else ViewUpdater(config) if plugin_path: diff --git a/src/plugins/compare/file_coverage/test/test_plugin_file_coverage.py b/src/plugins/compare/file_coverage/test/test_plugin_file_coverage.py index f9906db850..a0f9ab397c 100644 --- a/src/plugins/compare/file_coverage/test/test_plugin_file_coverage.py +++ b/src/plugins/compare/file_coverage/test/test_plugin_file_coverage.py @@ -25,7 +25,7 @@ def setup_plugin(self): This function must be overwritten by the test instance. In most cases it is sufficient to copy this function. ''' - return ComparePlugin(self, config=self.config, db_interface=DbMock(), view_updater=CommonDatabaseMock()) + return ComparePlugin(config=self.config, db_interface=DbMock(), view_updater=CommonDatabaseMock()) def test_get_intersection_of_files(self): self.fw_one.list_of_all_included_files.append('foo') diff --git a/src/scheduler/analysis.py b/src/scheduler/analysis.py index 415d428729..29677e4f3c 100644 --- a/src/scheduler/analysis.py +++ b/src/scheduler/analysis.py @@ -184,7 +184,7 @@ def _load_plugins(self): # be missing dependencies. So if anything goes wrong we want to inform the user about it logging.error(f'Could not import plugin {plugin_name} due to exception', exc_info=True) else: - self.analysis_plugins[plugin.AnalysisPlugin.NAME] = plugin.AnalysisPlugin(self, config=self.config) + self.analysis_plugins[plugin.AnalysisPlugin.NAME] = plugin.AnalysisPlugin(config=self.config) def _get_plugin_sets_from_config(self): try: diff --git a/src/test/unit/analysis/analysis_plugin_test_class.py b/src/test/unit/analysis/analysis_plugin_test_class.py index 73bfebe1d9..bb2f2e22b0 100644 --- a/src/test/unit/analysis/analysis_plugin_test_class.py +++ b/src/test/unit/analysis/analysis_plugin_test_class.py @@ -1,5 +1,6 @@ import unittest.mock from configparser import ConfigParser +from typing import Callable from test.common_helper import ( # pylint: disable=wrong-import-order CommonDatabaseMock, @@ -15,7 +16,7 @@ class AnalysisPluginTest(unittest.TestCase): # must be set by individual plugin test class PLUGIN_NAME = 'plugin_test' - PLUGIN_CLASS = None + PLUGIN_CLASS: Callable = None def setUp(self): self.docker_mount_base_dir = create_docker_mount_base_dir() @@ -28,9 +29,7 @@ def _set_config(self): def setup_plugin(self): # overwrite in plugin tests if necessary - return self.PLUGIN_CLASS( - self, config=self.config, view_updater=CommonDatabaseMock() - ) # pylint: disable=not-callable + return self.PLUGIN_CLASS(config=self.config, view_updater=CommonDatabaseMock()) def tearDown(self): self.analysis_plugin.shutdown() # pylint: disable=no-member @@ -50,13 +49,3 @@ def init_basic_config(self): config.set('data-storage', 'postgres-database', 'fact-test') return config - - def register_plugin(self, name, plugin_object): - ''' - This is a mock checking if the plugin registers correctly - ''' - self.assertEqual(name, self.PLUGIN_NAME, 'plugin registers with wrong name') - self.assertEqual(plugin_object.NAME, self.PLUGIN_NAME, 'plugin object has wrong name') - self.assertIsInstance(plugin_object.DESCRIPTION, str) - self.assertIsInstance(plugin_object.VERSION, str) - self.assertNotEqual(plugin_object.VERSION, 'not set', 'Plug-in version not set') diff --git a/src/test/unit/analysis/test_plugin_base.py b/src/test/unit/analysis/test_plugin_base.py index 5e6184789a..bac06f17d0 100644 --- a/src/test/unit/analysis/test_plugin_base.py +++ b/src/test/unit/analysis/test_plugin_base.py @@ -21,7 +21,7 @@ class TestPluginBase(unittest.TestCase): @mock.patch('plugins.base.ViewUpdater', lambda *_: None) def setUp(self): self.config = self.set_up_base_config() - self.base_plugin = DummyPlugin(self, self.config) + self.base_plugin = DummyPlugin(self.config) @staticmethod def set_up_base_config(): @@ -36,19 +36,12 @@ def tearDown(self): self.base_plugin.shutdown() gc.collect() - def register_plugin(self, name, plugin_object): # pylint: disable=no-self-use - ''' - This is a mock checking if the plugin registers correctly - ''' - assert name == 'dummy_plugin_for_testing_only', 'plugin registers with wrong name' - assert plugin_object.NAME == 'dummy_plugin_for_testing_only', 'plugin object has wrong name' - class TestPluginBaseCore(TestPluginBase): @mock.patch('plugins.base.ViewUpdater', lambda *_: None) def test_attribute_check(self): with pytest.raises(PluginInitException): - AnalysisBasePlugin(self, config=self.config) + AnalysisBasePlugin(config=self.config) @staticmethod def test_start_stop_workers(): @@ -134,7 +127,7 @@ def tearDown(self): @mock.patch('plugins.base.ViewUpdater', lambda *_: None) def multithread_config_test(self, multithread_flag, threads_in_config, threads_wanted): self.config.set('dummy_plugin_for_testing_only', 'threads', threads_in_config) - self.p_base = DummyPlugin(self, self.config, no_multithread=multithread_flag) + self.p_base = DummyPlugin(self.config, no_multithread=multithread_flag) self.assertEqual( self.p_base.config[self.p_base.NAME]['threads'], threads_wanted, 'number of threads not correct' ) @@ -148,7 +141,7 @@ def test_normal_multithread(self): @mock.patch('plugins.base.ViewUpdater', lambda *_: None) def test_init_result_dict(self): - self.p_base = DummyPlugin(self, self.config) + self.p_base = DummyPlugin(self.config) resultdict = self.p_base.init_dict() self.assertIn('analysis_date', resultdict, 'analysis date missing') self.assertEqual(resultdict['plugin_version'], '0.0', 'plugin version field not correct') @@ -166,12 +159,9 @@ def tearDown(self): @mock.patch('plugins.base.ViewUpdater', lambda *_: None) @mock.patch('plugins.analysis.dummy.code.dummy.AnalysisPlugin.TIMEOUT', 0) def test_timeout(self): - self.p_base = DummyPlugin(self, self.config) + self.p_base = DummyPlugin(self.config) fo_in = FileObject(binary=b'test', scheduled_analysis=[]) self.p_base.add_job(fo_in) fo_out = self.p_base.out_queue.get(timeout=5) self.p_base.shutdown() self.assertNotIn('summary', fo_out.processed_analysis['dummy_plugin_for_testing_only']) - - def register_plugin(self, name, plugin_object): - pass diff --git a/src/test/unit/compare/compare_plugin_test_class.py b/src/test/unit/compare/compare_plugin_test_class.py index a5b01ef4f8..5de9a4af87 100644 --- a/src/test/unit/compare/compare_plugin_test_class.py +++ b/src/test/unit/compare/compare_plugin_test_class.py @@ -26,7 +26,7 @@ def setup_plugin(self): ''' This function can be overwritten by the test instance. ''' - return self.PLUGIN_CLASS(self, config=self.config, view_updater=CommonDatabaseMock()) + return self.PLUGIN_CLASS(config=self.config, view_updater=CommonDatabaseMock()) def generate_config(self): # pylint: disable=no-self-use ''' @@ -34,16 +34,6 @@ def generate_config(self): # pylint: disable=no-self-use ''' return ConfigParser() - def test_init(self): - assert len(self.compare_plugins) == 1, 'number of registered plugins not correct' - assert self.compare_plugins[self.PLUGIN_NAME].NAME == self.PLUGIN_NAME, 'plugin instance not correct' - - def register_plugin(self, plugin_name, plugin_instance): - ''' - Callback Function Mock - ''' - self.compare_plugins[plugin_name] = plugin_instance - def setup_test_fw(self): self.fw_one = create_test_firmware(device_name='dev_1', all_files_included_set=True) self.fw_two = create_test_firmware( diff --git a/src/test/unit/compare/test_plugin_base.py b/src/test/unit/compare/test_plugin_base.py index 4a26e18250..ab9d866c53 100644 --- a/src/test/unit/compare/test_plugin_base.py +++ b/src/test/unit/compare/test_plugin_base.py @@ -18,7 +18,7 @@ def setup_plugin(self): This function must be overwritten by the test instance. In most cases it is sufficient to copy this function. """ - return ComparePlugin(self, config=self.config) + return ComparePlugin(config=self.config) def test_compare_missing_dep(self): self.c_plugin.DEPENDENCIES = ['test_ana'] From e10a823514aa8621115599f374399cb08376b039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 8 Nov 2022 16:49:24 +0100 Subject: [PATCH 10/16] fixed (node) installation for bionic --- src/install/pre_install.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/install/pre_install.sh b/src/install/pre_install.sh index 91ebd9a06b..dedf9701d3 100755 --- a/src/install/pre_install.sh +++ b/src/install/pre_install.sh @@ -73,7 +73,12 @@ sudo usermod -aG docker "$FACTUSER" # Setup npm repository as described in https://github.com/nodesource/distributions/blob/master/README.md#debinstall # This is required because the npm version that ships with Ubuntu 18.04 (bionic) is too old. -curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - +if [ "${CODENAME}" = "bionic" ]; then + # the latest LTS release is too new for bionic (requires glibc 2.28), the next most recent one is 14 + curl -fsSL https://deb.nodesource.com/setup_14.x | sudo -E bash - +else + curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - +fi IS_VENV=$(python3 -c 'import sys; print(sys.exec_prefix!=sys.base_prefix)') SUDO="" From e38ce6dbb0eafd1a32ae097cbb423e3daf2823f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Thu, 10 Nov 2022 16:31:15 +0100 Subject: [PATCH 11/16] refactored CI action with matrix --- .github/workflows/build_ci.yml | 134 ++------------------------------- 1 file changed, 5 insertions(+), 129 deletions(-) diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml index 095fcb962e..2c3f7f235e 100644 --- a/.github/workflows/build_ci.yml +++ b/.github/workflows/build_ci.yml @@ -6,8 +6,11 @@ on: workflow_dispatch: jobs: - focal: - runs-on: [self-hosted, linux, x64, focal] + build-ci: + strategy: + matrix: + os: [focal, jammy, buster, bullseye] + runs-on: [self-hosted, linux, x64, ${{ matrix.os }}] timeout-minutes: 45 steps: - name: Add Masks @@ -47,130 +50,3 @@ jobs: run: | python3 -m pip install codecov pytest --cov=. - - jammy: - runs-on: [self-hosted, linux, x64, jammy] - timeout-minutes: 45 - steps: - - name: Add Masks - run: | - echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" - echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" - echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" - - name: Checkout Branch - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - name: Set ulimit - run: ulimit -n 9999 - - name: Pre-Installation - shell: 'script -q -e -c "bash {0}"' - run: ./src/install/pre_install.sh - - name: Install FACT - shell: 'script -q -e -c "bash {0}"' - run: | - ./src/install.py -U -R -N -L DEBUG - - name: Unit Tests - shell: 'script -q -e -c "bash {0}"' - run: | - python3 -m pip install codecov - pytest --cov=. - - buster: - runs-on: [self-hosted, linux, x64, buster] - timeout-minutes: 45 - steps: - - name: Add Masks - run: | - echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" - echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" - echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" - - name: Checkout Branch - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - name: Set ulimit - run: ulimit -n 9999 - - name: Pre-Installation - shell: 'script -q -e -c "bash {0}"' - run: ./src/install/pre_install.sh - - name: Install FACT - shell: 'script -q -e -c "bash {0}"' - run: | - ./src/install.py -U -R -N -L DEBUG - - name: Unit Tests - shell: 'script -q -e -c "bash {0}"' - run: | - python3 -m pip install codecov - pytest --cov=. - - bullseye: - runs-on: [self-hosted, linux, x64, bullseye] - timeout-minutes: 45 - steps: - - name: Add Masks - run: | - echo "::add-mask::${{ secrets.NPM_REGISTRY_URL }}" - echo "::add-mask::${{ secrets.NPM_REGISTRY_AUTH }}" - echo "::add-mask::${{ secrets.CODECOV_TOKEN }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_1 }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_2 }}" - echo "::add-mask::${{ secrets.NETWORK_MASK_3 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_1 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_2 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_3 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_4 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_5 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" - echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" - - name: Checkout Branch - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - name: Set ulimit - run: ulimit -n 9999 - - name: Pre-Installation - shell: 'script -q -e -c "bash {0}"' - run: ./src/install/pre_install.sh - - name: Install FACT - shell: 'script -q -e -c "bash {0}"' - run: | - ./src/install.py -U -R -N -L DEBUG - - name: Unit Tests - shell: 'script -q -e -c "bash {0}"' - run: | - python3 -m pip install codecov - pytest --cov=. - From b516583b0a7fc539b68e6789c52e1244ef42e9bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Thu, 10 Nov 2022 16:36:43 +0100 Subject: [PATCH 12/16] fixed syntax error in CI action --- .github/workflows/build_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml index 2c3f7f235e..5a7634b841 100644 --- a/.github/workflows/build_ci.yml +++ b/.github/workflows/build_ci.yml @@ -9,8 +9,8 @@ jobs: build-ci: strategy: matrix: - os: [focal, jammy, buster, bullseye] - runs-on: [self-hosted, linux, x64, ${{ matrix.os }}] + os: [ focal, jammy, buster, bullseye ] + runs-on: [ self-hosted, linux, x64, "${{ matrix.os }}" ] timeout-minutes: 45 steps: - name: Add Masks From e8bb5a6ebc9d6f0f9bf07e7316b4e9b67df6c990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Thu, 10 Nov 2022 16:48:03 +0100 Subject: [PATCH 13/16] secret mask typo fix --- .github/workflows/build_ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml index 5a7634b841..03a9a4f9da 100644 --- a/.github/workflows/build_ci.yml +++ b/.github/workflows/build_ci.yml @@ -29,9 +29,9 @@ jobs: echo "::add-mask::${{ secrets.INTERNAL_NODE_6 }}" echo "::add-mask::${{ secrets.INTERNAL_NODE_7 }}" echo "::add-mask::${{ secrets.INTERNAL_NODE_8 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_1 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_2 }}" - echo "::add-mask::${{ secrets.SECRET_STRINGS_3 }}" + echo "::add-mask::${{ secrets.SECRET_STRING_1 }}" + echo "::add-mask::${{ secrets.SECRET_STRING_2 }}" + echo "::add-mask::${{ secrets.SECRET_STRING_3 }}" - name: Checkout Branch uses: actions/checkout@v3 with: From 74282d10958926197de401617d9dbd9168ed9616 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Nov 2022 13:42:03 +0000 Subject: [PATCH 14/16] build(deps): bump cryptography from 37.0.2 to 38.0.3 in /src/install Bumps [cryptography](https://github.com/pyca/cryptography) from 37.0.2 to 38.0.3. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/37.0.2...38.0.3) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- src/install/requirements_backend.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/install/requirements_backend.txt b/src/install/requirements_backend.txt index 0d4d063746..012b215857 100644 --- a/src/install/requirements_backend.txt +++ b/src/install/requirements_backend.txt @@ -1,4 +1,4 @@ -cryptography==37.0.2 +cryptography==38.0.3 docker==5.0.3 MarkupSafe==2.0.1 matplotlib==3.5.2 From 4b868f2c25b190336a1e49d8ae66a886d1dc778d Mon Sep 17 00:00:00 2001 From: rhelmke Date: Mon, 14 Nov 2022 15:21:15 +0100 Subject: [PATCH 15/16] disable fail-fast for ci build matrix --- .github/workflows/build_ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml index 03a9a4f9da..c748bea40a 100644 --- a/.github/workflows/build_ci.yml +++ b/.github/workflows/build_ci.yml @@ -8,6 +8,7 @@ on: jobs: build-ci: strategy: + fail-fast: false matrix: os: [ focal, jammy, buster, bullseye ] runs-on: [ self-hosted, linux, x64, "${{ matrix.os }}" ] From 47f5fcf5d84eb3bd185a12981fc0196925f6e666 Mon Sep 17 00:00:00 2001 From: rhelmke Date: Tue, 15 Nov 2022 12:55:24 +0100 Subject: [PATCH 16/16] add time based ci build at 03:00am UTC each night --- .github/workflows/build_ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml index 03a9a4f9da..0254af6faf 100644 --- a/.github/workflows/build_ci.yml +++ b/.github/workflows/build_ci.yml @@ -3,6 +3,8 @@ run-name: Build CI on: pull_request: branches: [ master ] + schedule: + - cron: "0 3 * * *" workflow_dispatch: jobs: