Merge branch 'master' into remove_legacy_collection

.flake8

@@ -1,3 +1,3 @@
 [flake8]
 ignore = E501,E126,E127,E128,E722,E741,W504
-exclude = insights/contrib,bin,docs,include,lib,lib64,.git,.collections.py,insights/parsers/tests/lvm_test_data.py,insights/client/apps/ansible/playbook_verifier/contrib
+exclude = insights/contrib,bin,docs,include,lib,lib64,.git,.collections.py,insights/parsers/tests/lvm_test_data.py,insights/client/apps/ansible/playbook_verifier/contrib,insights/tools/coverage.py

.github/pull_request_template.md

@@ -3,6 +3,7 @@
 Check all that apply:
 
 * [ ] Have you followed the guidelines in our Contributing document, including the instructions about commit messages?
+* [ ] No Sensitive Data in this change?
 * [ ] Is this PR to correct an issue?
 * [ ] Is this PR an enhancement?
 

.github/workflows/main.yml

@@ -28,11 +28,18 @@ jobs:
       run: |
         pip install -e .[linting]
         flake8 .
-    - name: pytest
+    - name: pytest with coverage report
       run: |
-        pip install 'urllib3'
+        pip install urllib3
         pip install -e .[testing]
-        pytest
+        pytest --cov --cov-branch --cov-report=
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        verbose: true
+        flags: unittests
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   python27-test:
 
@@ -53,7 +60,7 @@ jobs:
         flake8 .
     - name: pytest
       run: |
-        pip install 'urllib3'
+        pip install urllib3
         pip install -e .[testing]
         pytest
 
@@ -125,3 +132,17 @@ jobs:
       run: |
         pip install -e .[docs]
         sphinx-build -W -b html -qa -E docs docs/_build/html
+
+  gitleaks:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    # - uses: gitleaks/gitleaks-action@v2
+    - uses: gitleaks/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }}

insights/collect.py

@@ -167,13 +167,6 @@
         - name: insights.combiners.ps
           enabled: true
 
-    # needed for httpd_certificate
-        - name: insights.combiners.httpd_conf.HttpdConfTree
-          enabled: true
-
-        - name: insights.parsers.httpd_conf.HttpdConf
-          enabled: true
-
     # needed for httpd_on_nfs
         - name: insights.parsers.mount.ProcMounts
           enabled: true

insights/combiners/lspci.py

@@ -122,8 +122,8 @@ def __init__(self, lspci_k, lspci_vmmkn):
         if lspci_vmmkn:
             for dev in lspci_vmmkn:
                 # use the local copy to prevent from writing back to the parser
-                dev = dev.copy()
-                if lspci_k and dev['Slot'] in lspci_k:
+                dev_t = dev.copy()
+                if lspci_k and (dev['Slot'] in lspci_k.data or '0000:' + dev['Slot'] in lspci_k.data):
                     # use the local copy to prevent from writing back to the parser
                     dev_k = [v for v in lspci_k.data.values() if v['Slot'].endswith(dev['Slot'])][0].copy()
                     # Since the 'lspci -k' is a more common command than the
@@ -133,8 +133,9 @@ def __init__(self, lspci_k, lspci_vmmkn):
                     # dev_k.pop('Slot') if 'Slot' in dev_k else None
                     dev_k.pop('Kernel driver in use') if 'Kernel driver in use' in dev_k else None
                     dev_k.pop('Kernel modules') if 'Kernel modules' in dev_k else None
-                    dev.update(dev_k)
-                self.append(dev)
+                    dev_t.update(dev_k)
+
+                self.append(dev_t)
             self._pci_dev_list = (lspci_k if lspci_k else lspci_vmmkn).pci_dev_list
         else:
             for dev in lspci_k.data.values():

insights/parsers/cups_confs.py

@@ -6,6 +6,8 @@
 
 CupsdConf - file ``/etc/cups/cupsd.conf``
 -----------------------------------------
+CupsBrowsedConf - file ``/etc/cups/cups-browsed.conf``
+------------------------------------------------------
 CupsFilesConf - file ``/etc/cups/cups-files.conf``
 --------------------------------------------------
 """
@@ -90,6 +92,43 @@ def parse_doc(self, content):
         return Entry(children=result, src=self)
 
 
+@parser(Specs.cups_browsed_conf)
+class CupsBrowsedConf(Parser, dict):
+    """
+    Class for parsing the file ``/etc/cups/cups-browsed.conf``
+
+    .. note::
+
+        The admin can add multiple directives into the configuration file, and restart service without issue.
+        However, item like BrowseRemoteProtocols only works for the last one, and the item like BrowseAllow works
+        for all directives. So the values of each directives will get stored to a list with the original order,
+        and duplicated values will be kept without de-duplication.
+
+    Sample file content::
+
+        BrowseRemoteProtocols dnssd cups
+        BrowseAllow cups.example.com
+
+    Examples:
+        >>> type(cups_browsed_conf)
+        <class 'insights.parsers.cups_confs.CupsBrowsedConf'>
+        >>> 'dnssd cups' in cups_browsed_conf['BrowseRemoteProtocols']
+        True
+        >>> 'cups.example.com' in cups_browsed_conf['BrowseAllow']
+        True
+    """
+    def parse_content(self, content):
+        if not content:
+            raise ParseException('Empty Content')
+
+        for line in get_active_lines(content):
+            k, v = [i.strip() for i in line.split(None, 1)]
+            if k not in self:
+                self[k] = [v]
+            else:
+                self[k].append(v)
+
+
 @parser(Specs.cups_files_conf)
 class CupsFilesConf(Parser, dict):
     """

insights/parsers/sysconfig.py

@@ -36,6 +36,9 @@
 KdumpSysconfig - file ``/etc/sysconfig/kdump``
 ----------------------------------------------
 
+KernelSysconfig - file ``/etc/sysconfig/kernel``
+------------------------------------------------
+
 LibvirtGuestsSysconfig - file ``/etc/sysconfig/libvirt-guests``
 ---------------------------------------------------------------
 
@@ -350,6 +353,29 @@ def parse_content(self, content):
             setattr(self, key, self.data.get(key, ''))
 
 
+@parser(Specs.sysconfig_kernel)
+class KernelSysconfig(SysconfigOptions):
+    """
+    This parser reads data from the ``/etc/sysconfig/kernel`` file.
+
+    Typical content example::
+
+        # UPDATEDEFAULT specifies if new-kernel-pkg should make
+        # new kernels the default
+        UPDATEDEFAULT=yes
+
+        # DEFAULTKERNEL specifies the default kernel package type
+        DEFAULTKERNEL=kernel
+
+    Examples:
+        >>> kernel_syscfg.get('UPDATEDEFAULT')
+        'yes'
+        >>> 'DEFAULTKERNEL' in kernel_syscfg
+        True
+    """
+    pass
+
+
 @parser(Specs.sysconfig_libvirt_guests)
 class LibvirtGuestsSysconfig(SysconfigOptions):
     """

insights/parsers/uname.py

@@ -36,10 +36,12 @@
     False
 
 """
+import warnings
 
 from collections import namedtuple
 from distutils.version import LooseVersion, StrictVersion
-from .. import parser, CommandParser
+
+from insights import parser, CommandParser
 from insights.core.context import Context
 from insights.specs import Specs
 
@@ -416,8 +418,7 @@ def _best_lv_release(self, other):
         is_o_with_dist = o_release_parts[-1].startswith(dist_opts)
         if not (is_s_with_dist ^ is_o_with_dist):
             return self._lv_release, other._lv_release
-        import warnings
-        warnings.warn('Comparison of distribution part will be ingored.')
+        warnings.warn('Comparison of distribution part will be ignored.')
         s_release = (
             self._lv_release.vstring if not is_s_with_dist
             else pad_release(".".join(s_release_parts[:-1]), len(s_release_parts))

insights/specs/__init__.py

@@ -91,7 +91,6 @@ class Specs(SpecSet):
     cinder_volume_log = RegistryPoint(filterable=True)
     cloud_cfg = RegistryPoint(filterable=True)
     cloud_cfg_filtered = RegistryPoint()
-    cloud_init_cfg_run = RegistryPoint()
     cloud_init_custom_network = RegistryPoint()
     cloud_init_log = RegistryPoint(filterable=True)
     cluster_conf = RegistryPoint(filterable=True)
@@ -125,6 +124,7 @@ class Specs(SpecSet):
     crypto_policies_state_current = RegistryPoint(no_obfuscate=['hostname', 'ip'])
     cryptsetup_luksDump = RegistryPoint(multi_output=True, no_obfuscate=['hostname', 'ip'])
     cupsd_conf = RegistryPoint()
+    cups_browsed_conf = RegistryPoint(filterable=True)
     cups_files_conf = RegistryPoint()
     cups_ppd = RegistryPoint(multi_output=True, no_obfuscate=['hostname', 'ip'])
     current_clocksource = RegistryPoint(no_obfuscate=['hostname', 'ip'])
@@ -757,6 +757,7 @@ class Specs(SpecSet):
     sysconfig_httpd = RegistryPoint()
     sysconfig_irqbalance = RegistryPoint()
     sysconfig_kdump = RegistryPoint()
+    sysconfig_kernel = RegistryPoint()
     sysconfig_libvirt_guests = RegistryPoint()
     sysconfig_memcached = RegistryPoint()
     sysconfig_mongod = RegistryPoint(multi_output=True)

insights/specs/datasources/httpd.py

@@ -71,8 +71,14 @@ def _get_all_include_conf(root, glob_path):
                 _paths.add(conf)
                 with open(conf) as cfp:
                     _includes = None
+                    section_number = 0
                     for line in cfp.readlines():
-                        if line.strip().startswith("Include"):
+                        line = line.strip()
+                        if line.startswith("</"):
+                            section_number = section_number - 1
+                        elif line.startswith("<"):
+                            section_number = section_number + 1
+                        if section_number == 0 and (line.startswith("Include ") or line.startswith("IncludeOptional ")):
                             _includes = line.split()[-1].strip('"\'')
                             _paths.update(_get_all_include_conf(root, _includes))
             if os.path.isdir(conf):
@@ -92,11 +98,17 @@ def get_httpd_configuration_files(httpd_root):
             server_root = httpd_root
             # Add it only when it exists
             all_paths.add(main_httpd_conf)
+            section_number = 0
             for line in cfp.readlines():
-                if line.strip().startswith("ServerRoot"):
+                line = line.strip()
+                if line.startswith("</"):
+                    section_number = section_number - 1
+                elif line.startswith("<"):
+                    section_number = section_number + 1
+                if line.startswith("ServerRoot "):
                     server_root = line.strip().split()[-1].strip().strip('"\'')
-                elif line.strip().startswith("Include"):
-                    includes = line.strip().split()[-1].strip('"\'')
+                elif section_number == 0 and (line.startswith("Include ") or line.startswith("IncludeOptional ")):
+                    includes = line.split()[-1].strip('"\'')
                     # For multiple "Include" directives, all of them will be included
                     all_paths.update(_get_all_include_conf(server_root, includes))
     except Exception:
@@ -117,10 +129,7 @@ def httpd_configuration_files(broker):
         SkipComponent: there is no httpd configuration file
     """
     httpd_root = '/etc/httpd'
-    all_paths = get_httpd_configuration_files(httpd_root)
-    if all_paths:
-        return all_paths
-    raise SkipComponent
+    return get_httpd_configuration_files(httpd_root)
 
 
 @datasource(HostContext)
@@ -135,10 +144,7 @@ def httpd24_scl_configuration_files(broker):
         SkipComponent: there is no httpd24 slc configuration file
     """
     httpd_root = '/opt/rh/httpd24/root/etc/httpd'
-    all_paths = get_httpd_configuration_files(httpd_root)
-    if all_paths:
-        return all_paths
-    raise SkipComponent
+    return get_httpd_configuration_files(httpd_root)
 
 
 @datasource(HostContext)
@@ -153,7 +159,4 @@ def httpd24_scl_jbcs_configuration_files(broker):
         SkipComponent: there is no httpd24 slc jbcs configuration file
     """
     httpd_root = '/opt/rh/jbcs-httpd24/root/etc/httpd'
-    all_paths = get_httpd_configuration_files(httpd_root)
-    if all_paths:
-        return all_paths
-    raise SkipComponent
+    return get_httpd_configuration_files(httpd_root)