diff --git a/health-check/src/uyuni_health_check/config.ini b/health-check/src/uyuni_health_check/config.ini new file mode 100644 index 00000000000..c9307e570d1 --- /dev/null +++ b/health-check/src/uyuni_health_check/config.ini @@ -0,0 +1,10 @@ +[podman] +network_name = "health-check-network" + +[loki] +loki_container_name = uyuni_health_check_loki +loki_port = 3100 + +[logcli] +logcli_container_name = uyuni_health_check_logcli +logcli_image_name = logcli diff --git a/health-check/src/uyuni_health_check/config/exporter/config.yaml b/health-check/src/uyuni_health_check/config/exporter/config.yaml index 682f7383642..8f312ca04cc 100644 --- a/health-check/src/uyuni_health_check/config/exporter/config.yaml +++ b/health-check/src/uyuni_health_check/config/exporter/config.yaml @@ -1,3 +1,3 @@ -supportconfig_path: /home/ygutierrez/Documents/L3/1227859/scc_lxms-cp-suma01_240828_1131 +supportconfig_path: /home/ygutierrez/Documents/L3/1230944/scc_s3p3238_240916_1049 port: 9000 scrape_frequency: 60 \ No newline at end of file diff --git a/health-check/src/uyuni_health_check/config/promtail/config.yaml b/health-check/src/uyuni_health_check/config/promtail/config.yaml index 89da526ae73..3cae492398d 100644 --- a/health-check/src/uyuni_health_check/config/promtail/config.yaml +++ b/health-check/src/uyuni_health_check/config/promtail/config.yaml @@ -15,7 +15,7 @@ scrape_configs: - localhost labels: job: cobbler - __path__: /home/ygutierrez/Documents/L3/1227859/scc_lxms-cp-suma01_240828_1131/spacewalk-debug/cobbler-logs/cobbler.log + __path__: /home/ygutierrez/Documents/L3/1230944/scc_s3p3238_240916_1049/spacewalk-debug/cobbler-logs/cobbler.log pipeline_stages: - multiline: firstline: '^\[.*\] [\dT:-]+ - [A-Z]+ | ' @@ -34,7 +34,7 @@ scrape_configs: - localhost labels: job: postgresql - __path__: /home/ygutierrez/Documents/L3/1227859/scc_lxms-cp-suma01_240828_1131/spacewalk-debug/database//postgresql*.log* + __path__: /home/ygutierrez/Documents/L3/1230944/scc_s3p3238_240916_1049/spacewalk-debug/database//postgresql*.log* pipeline_stages: - multiline: firstline: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \S+ ' @@ -53,7 +53,7 @@ scrape_configs: - localhost labels: job: salt - __path__: /home/ygutierrez/Documents/L3/1227859/scc_lxms-cp-suma01_240828_1131/spacewalk-debug/salt-logs/salt/{api,master} + __path__: /home/ygutierrez/Documents/L3/1230944/scc_s3p3238_240916_1049/spacewalk-debug/salt-logs/salt/{api,master} pipeline_stages: - multiline: firstline: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(,)\d{3} ' @@ -89,13 +89,13 @@ scrape_configs: labels: job: rhn component: tomcat - __path__: /home/ygutierrez/Documents/L3/1227859/scc_lxms-cp-suma01_240828_1131/spacewalk-debug/rhn-logs/rhn/rhn_{salt,web}_*log + __path__: /home/ygutierrez/Documents/L3/1230944/scc_s3p3238_240916_1049/spacewalk-debug/rhn-logs/rhn/rhn_{salt,web}_*log - targets: - localhost labels: job: rhn component: taskomatic - __path__: /home/ygutierrez/Documents/L3/1227859/scc_lxms-cp-suma01_240828_1131/spacewalk-debug/rhn-logs/rhn/rhn_taskomatic_*log + __path__: /home/ygutierrez/Documents/L3/1230944/scc_s3p3238_240916_1049/spacewalk-debug/rhn-logs/rhn/rhn_taskomatic_*log - job_name: apache static_configs: - targets: @@ -103,19 +103,19 @@ scrape_configs: labels: job: apache component: error_log - __path__: /home/ygutierrez/Documents/L3/1227859/scc_lxms-cp-suma01_240828_1131/spacewalk-debug/httpd-logs/apache2/error_log* + __path__: /home/ygutierrez/Documents/L3/1230944/scc_s3p3238_240916_1049/spacewalk-debug/httpd-logs/apache2/error_log* - targets: - localhost labels: job: apache component: access_log - __path__: /home/ygutierrez/Documents/L3/1227859/scc_lxms-cp-suma01_240828_1131/spacewalk-debug/httpd-logs/apache2/access_log* + __path__: /home/ygutierrez/Documents/L3/1230944/scc_s3p3238_240916_1049/spacewalk-debug/httpd-logs/apache2/access_log* - targets: - localhost labels: job: apache component: ssl_request_log - __path__: /home/ygutierrez/Documents/L3/1227859/scc_lxms-cp-suma01_240828_1131/spacewalk-debug/httpd-logs/apache2/ssl_request_log* + __path__: /home/ygutierrez/Documents/L3/1230944/scc_s3p3238_240916_1049/spacewalk-debug/httpd-logs/apache2/ssl_request_log* pipeline_stages: - match: selector: '{job="apache",component="error_log"} |~ "^\\[[A-Z][a-z]{2} "' diff --git a/health-check/src/uyuni_health_check/config_loader.py b/health-check/src/uyuni_health_check/config_loader.py index 6044547dbf6..b240880c541 100644 --- a/health-check/src/uyuni_health_check/config_loader.py +++ b/health-check/src/uyuni_health_check/config_loader.py @@ -1,8 +1,8 @@ import os +import configparser import yaml from jinja2 import Environment, FileSystemLoader - class ConfigLoader: def __init__(self): self.base_dir = os.path.dirname(os.path.abspath(__file__)) @@ -10,6 +10,12 @@ def __init__(self): self.config_dir = os.path.join(self.base_dir, "config") self.containers_dir = os.path.join(self.base_dir, "containers") self.jinja_env = Environment(loader=FileSystemLoader(self.templates_dir)) + self.load_config() + + + def load_config(self, file_path='config.ini'): + self.global_config = configparser.ConfigParser() + self.global_config.read(os.path.join(self.base_dir, "config.ini")) def load_yaml(self, filename): file_path = os.path.join(self.config_dir, filename) diff --git a/health-check/src/uyuni_health_check/containers/exporter/supportconfig_exporter.py b/health-check/src/uyuni_health_check/containers/exporter/supportconfig_exporter.py index 5ff9626f98f..a7adc31d66c 100644 --- a/health-check/src/uyuni_health_check/containers/exporter/supportconfig_exporter.py +++ b/health-check/src/uyuni_health_check/containers/exporter/supportconfig_exporter.py @@ -30,9 +30,12 @@ def __init__(self, supportconfig_path=None): self.refresh() def refresh(self): - self.salt_jobs = self.read_salt_jobs() - self.salt_keys = self.read_salt_keys() - self.salt_configuration = self.read_salt_configuration() + if self.exists_salt_jobs_file(): + self.salt_jobs = self.read_salt_jobs() + if self.exists_salt_keys_file(): + self.salt_keys = self.read_salt_keys() + if self.exists_salt_configuration_file(): + self.salt_configuration = self.read_salt_configuration() def _parse_command(self, command_block): lines = command_block.strip().split("\n") @@ -61,6 +64,10 @@ def parse_supportconfig_plugin_file(self, filein): ret[cmd] = val return ret + def exists_salt_configuration_file(self): + if not os.path.isfile(os.path.join(self.supportconfig_path, "plugin-saltconfiguration.txt")): + return + def read_salt_configuration(self): content = None with open( @@ -77,6 +84,10 @@ def read_salt_configuration(self): for attr in attrs_to_expose: ret[attr] = re.findall(f"^{attr}: ([0-9]+)$", content, re.MULTILINE)[-1] return ret + + def exists_salt_keys_file(self): + if not os.path.isfile(os.path.join(self.supportconfig_path, "plugin-saltminionskeys.txt")): + return def read_salt_keys(self): content = None @@ -96,6 +107,10 @@ def read_salt_keys(self): ret["unaccepted"] = parsed[2].strip().split("\n") if parsed[2].strip() else [] ret["rejected"] = parsed[3].strip().split("\n") if parsed[3].strip() else [] return ret + + def exists_salt_jobs_file(self): + if not os.path.isfile(os.path.join(self.supportconfig_path, "plugin-saltjobs.txt")): + return def read_salt_jobs(self): content = None @@ -109,40 +124,44 @@ def collect(self): gauge = GaugeMetricFamily( "salt_master_config", "Salt Master Configuration", labels=["name"] ) - gauge.add_metric( - ["worker_threads"], - self.salt_configuration["worker_threads"], - ) - gauge.add_metric( - ["sock_pool_size"], - self.salt_configuration["sock_pool_size"], - ) - gauge.add_metric(["timeout"], self.salt_configuration["timeout"]) - gauge.add_metric( - ["gather_job_timeout"], - self.salt_configuration["gather_job_timeout"], - ) - yield gauge - - gauge2 = GaugeMetricFamily( - "salt_keys", - "Information about Salt keys", - labels=["name"], - ) - gauge2.add_metric(["accepted"], len(self.salt_keys["accepted"])) - gauge2.add_metric(["denied"], len(self.salt_keys["denied"])) - gauge2.add_metric(["unaccepted"], len(self.salt_keys["unaccepted"])) - gauge2.add_metric(["rejected"], len(self.salt_keys["rejected"])) - yield gauge2 - - gauge3 = GaugeMetricFamily( - "salt_jobs", - "Information about Salt Jobs", - labels=["jid", "fun"], - ) - for jid, fun in self.salt_jobs: - gauge3.add_metric([jid, fun], 1) - yield gauge3 + if hasattr(self,'salt_configuration'): + gauge.add_metric( + ["worker_threads"], + self.salt_configuration["worker_threads"], + ) + + gauge.add_metric( + ["sock_pool_size"], + self.salt_configuration["sock_pool_size"], + ) + gauge.add_metric(["timeout"], self.salt_configuration["timeout"]) + gauge.add_metric( + ["gather_job_timeout"], + self.salt_configuration["gather_job_timeout"], + ) + yield gauge + + if hasattr(self,'salt_keys'): + gauge2 = GaugeMetricFamily( + "salt_keys", + "Information about Salt keys", + labels=["name"], + ) + gauge2.add_metric(["accepted"], len(self.salt_keys["accepted"])) + gauge2.add_metric(["denied"], len(self.salt_keys["denied"])) + gauge2.add_metric(["unaccepted"], len(self.salt_keys["unaccepted"])) + gauge2.add_metric(["rejected"], len(self.salt_keys["rejected"])) + yield gauge2 + + if hasattr(self,'salt_jobs'): + gauge3 = GaugeMetricFamily( + "salt_jobs", + "Information about Salt Jobs", + labels=["jid", "fun"], + ) + for jid, fun in self.salt_jobs: + gauge3.add_metric([jid, fun], 1) + yield gauge3 def main(): diff --git a/health-check/src/uyuni_health_check/containers/manager.py b/health-check/src/uyuni_health_check/containers/manager.py index 54b52da4509..2002c55f806 100644 --- a/health-check/src/uyuni_health_check/containers/manager.py +++ b/health-check/src/uyuni_health_check/containers/manager.py @@ -15,7 +15,7 @@ def podman(cmd, quiet=True, use_print=False): """ try: if not quiet: - console.log(f"[italic]Running command {['podman'] + cmd}[/italic]") + console.log(f"[italic]Running command {'podman ' + ' '.join(cmd)}[/italic]") return run_command(["podman"] + cmd, console, quiet=quiet, use_print=use_print) except OSError: raise HealthException("podman is required") @@ -48,19 +48,16 @@ def image_exists(image): """ Check if the image is present in podman images result """ - return ( - podman(["images", "--quiet", "-f", f"reference={image}"], quiet=True) - .stdout.read() - .strip() - != "" - ) + stdout, stderr, _ = podman(["images", "--quiet", "-f", f"reference={image}"], quiet=True) + return stdout.strip() != "" def network_exists(network): """ Check if the podman network is up and running """ - return podman(["network", "exists", f"{network}"], quiet=True).returncode == 0 + stdout, stderr, returncode = podman(["network", "exists", f"{network}"], quiet=True) + return returncode == 0 def clean_containers(verbose=False): @@ -131,5 +128,5 @@ def container_is_running(name): """ Check if a container with a given name is running in podman """ - process = podman(["ps", "--quiet", "-f", f"name={name}"]) - return process.stdout.read() != "" + stdout, stderr, _ = podman(["ps", "--quiet", "-f", f"name={name}"]) + return stdout != "" diff --git a/health-check/src/uyuni_health_check/exporters/supportconfig/exporter.py b/health-check/src/uyuni_health_check/exporters/supportconfig/exporter.py index 66ad049638e..629a527badf 100644 --- a/health-check/src/uyuni_health_check/exporters/supportconfig/exporter.py +++ b/health-check/src/uyuni_health_check/exporters/supportconfig/exporter.py @@ -52,8 +52,6 @@ def prepare_exporter(config=None, verbose=False, supportconfig_path=None): f"{supportconfig_path}:{supportconfig_path}", "-v", f"{exporter_config}:/opt/config.yml", - "-p", - "9000:9000", ] podman_args.extend( @@ -63,7 +61,7 @@ def prepare_exporter(config=None, verbose=False, supportconfig_path=None): f"{exporter_name}", ] ) - + console.log(f"Running this command: podman " + ' '.join(podman_args)) # Run the container podman( podman_args, diff --git a/health-check/src/uyuni_health_check/exporters/supportconfig/supportconfig_exporter.py b/health-check/src/uyuni_health_check/exporters/supportconfig/supportconfig_exporter.py deleted file mode 100644 index 98e3ac276cd..00000000000 --- a/health-check/src/uyuni_health_check/exporters/supportconfig/supportconfig_exporter.py +++ /dev/null @@ -1,185 +0,0 @@ -# SPDX-FileCopyrightText: 2023 SUSE LLC -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import re -import signal -import sys -import time -import logging - -import yaml -from prometheus_client import start_http_server -from prometheus_client.core import REGISTRY, GaugeMetricFamily - - -def sigterm_handler(**kwargs): - print("Detected SIGTERM. Exiting.") - sys.exit(0) - - -signal.signal(signal.SIGTERM, sigterm_handler) - - -class SupportConfigMetricsCollector(object): - def __init__(self, supportconfig_path=None): - if not supportconfig_path: - raise ValueError("A 'supportconfig_path' must be set via config.yml file") - - self.supportconfig_path = supportconfig_path - self.refresh() - - def refresh(self): - self.salt_jobs = self.read_salt_jobs() - self.salt_keys = self.read_salt_keys() - self.salt_configuration = self.read_salt_configuration() - - def _parse_command(self, command_block): - lines = command_block.strip().split("\n") - command = lines[0][2:] - return command, lines[1:] - - def parse_supportconfig_file(self, filein): - content = None - with open(os.path.join(self.supportconfig_path, filein)) as f: - content = f.read() - parsed = re.findall(r"^#==\[ (.*) \]=+#$((?:\n.+)+)$", content, re.MULTILINE) - ret = {} - for _, value in parsed: - cmd, val = self._parse_command(value) - ret[cmd] = val - return ret - - def parse_supportconfig_plugin_file(self, filein): - content = None - with open(os.path.join(self.supportconfig_path, filein)) as f: - content = f.read() - parsed = re.findall(r"^#==\[ (.*) \]=+#$((?:\n.+)+)$", content, re.MULTILINE) - ret = {} - for _, value in parsed: - cmd, val = self._parse_command(value) - ret[cmd] = val - return ret - - def read_salt_configuration(self): - content = None - with open( - os.path.join(self.supportconfig_path, "plugin-saltconfiguration.txt") - ) as f: - content = f.read() - attrs_to_expose = [ - "worker_threads", - "sock_pool_size", - "timeout", - "gather_job_timeout", - ] - ret = {} - for attr in attrs_to_expose: - ret[attr] = re.findall(f"^{attr}: ([0-9]+)$", content, re.MULTILINE)[-1] - return ret - - def read_salt_keys(self): - content = None - with open( - os.path.join(self.supportconfig_path, "plugin-saltminionskeys.txt") - ) as f: - content = f.read() - ret = {} - parsed = re.findall( - r"^Accepted Keys:$((?:\n.*)*)\nDenied Keys:$((?:\n.*)*)\n" - r"Unaccepted Keys:$((?:\n.*)*)\nRejected Keys:$((?:\n.*)*)#==", - content, - re.MULTILINE, - )[0] - ret["accepted"] = parsed[0].strip().split("\n") if parsed[0].strip() else [] - ret["denied"] = parsed[1].strip().split("\n") if parsed[1].strip() else [] - ret["unaccepted"] = parsed[2].strip().split("\n") if parsed[2].strip() else [] - ret["rejected"] = parsed[3].strip().split("\n") if parsed[3].strip() else [] - return ret - - def read_salt_jobs(self): - content = None - with open(os.path.join(self.supportconfig_path, "plugin-saltjobs.txt")) as f: - content = f.read() - return re.findall( - "^'([0-9]+)':$(?:\n.*\n.*Function: (.*)\n[^'|#==]*)", content, re.MULTILINE - ) - - def collect(self): - gauge = GaugeMetricFamily( - "salt_master_config", "Salt Master Configuration", labels=["name"] - ) - gauge.add_metric( - ["worker_threads"], - self.salt_configuration["worker_threads"], - ) - gauge.add_metric( - ["sock_pool_size"], - self.salt_configuration["sock_pool_size"], - ) - gauge.add_metric(["timeout"], self.salt_configuration["timeout"]) - gauge.add_metric( - ["gather_job_timeout"], - self.salt_configuration["gather_job_timeout"], - ) - yield gauge - - gauge2 = GaugeMetricFamily( - "salt_keys", - "Information about Salt keys", - labels=["name"], - ) - gauge2.add_metric(["accepted"], len(self.salt_keys["accepted"])) - gauge2.add_metric(["denied"], len(self.salt_keys["denied"])) - gauge2.add_metric(["unaccepted"], len(self.salt_keys["unaccepted"])) - gauge2.add_metric(["rejected"], len(self.salt_keys["rejected"])) - yield gauge2 - - gauge3 = GaugeMetricFamily( - "salt_jobs", - "Information about Salt Jobs", - labels=["jid", "fun"], - ) - for jid, fun in self.salt_jobs: - gauge3.add_metric([jid, fun], 1) - yield gauge3 - - -def main(): - logging.basicConfig( - stream=sys.stdout, - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", - ) - - logging.info("STARTING SUPPORTCONFIG EXPORTER PROCESS") - print("Supportconfig Exporter started") - port = 9000 - frequency = 60 - supportconfig_path = None - if os.path.exists("config.yml"): - with open("config.yml", "r") as config_file: - try: - config = yaml.safe_load(config_file) - port = int(config["port"]) - frequency = config["scrape_frequency"] - supportconfig_path = config["supportconfig_path"] - except yaml.YAMLError as error: - print(error) - logging.info("Starting HTTP server") - start_http_server(port) - logging.info("Creating metrics collector object") - collector = SupportConfigMetricsCollector(supportconfig_path) - logging.info("Registering collector") - REGISTRY.register(collector) - logging.info("Supporconfig Exporter is ready") - while True: - # period between collection - time.sleep(frequency) - logging.info("Refreshing collector") - collector.refresh() - - -if __name__ == "__main__": - main() diff --git a/health-check/src/uyuni_health_check/loki/logs_gatherer.py b/health-check/src/uyuni_health_check/loki/logs_gatherer.py index 6bc79c4952c..e2ebfdc4bd7 100644 --- a/health-check/src/uyuni_health_check/loki/logs_gatherer.py +++ b/health-check/src/uyuni_health_check/loki/logs_gatherer.py @@ -1,17 +1,26 @@ from uyuni_health_check.containers.manager import podman +from uyuni_health_check.utils import run_command, HealthException, console +from uyuni_health_check.outputter import outputter +from config_loader import ConfigLoader from rich.markdown import Markdown from datetime import datetime, timedelta from rich import print +import json +from json.decoder import JSONDecodeError +conf = ConfigLoader() -def show_full_error_logs(since, loki=None): +def show_full_error_logs(from_datetime=None, to_datetime=None, since=7, loki=None): """ Get and show the error logs """ print() print(Markdown(f"- Getting error messages over the last {since} days...")) - from_time = (datetime.utcnow() - timedelta(days=since)).isoformat() - loki_url = loki or "http://uyuni_health_check_loki:3100" + #from_time = (datetime.utcnow() - timedelta(days=since)).isoformat() + #loki_url = loki or "http://uyuni_health_check_loki:3100" + loki_container_name = conf.global_config['loki']['loki_container_name'] + loki_port = conf.global_config['loki']['loki_port'] + loki_url = f"http://{loki_container_name}:{loki_port}" podman( [ "run", @@ -26,12 +35,87 @@ def show_full_error_logs(since, loki=None): "--quiet", "--output=jsonl", f"--addr={loki_url}", - f"--from={from_time}Z", + f"--from={from_datetime}", + f"--to={to_datetime}", "--limit=150", - # '"{job=~\".+\"}"', '{job=~".+"} |~ `(?i)error|(?i)severe|(?i)critical|(?i)fatal`', ], quiet=False, use_print=True, ) print() + + +def show_error_logs_stats(from_datetime, to_datetime, since, console: "Console", loki=None): + """ + Get and show the error logs stats + """ + print( + Markdown(f"- Getting summary of errors in logs") + ) + print() + query = f'count_over_time({{job=~".+"}} |~ "(?i)error|(?i)severe|(?i)critical|(?i)fatal" [{since}d])' + # Returns a JSON. + stdout, stderr = query_loki(from_dt=from_datetime, to_dt=to_datetime, since = since, query=query) + + try: + data = json.loads(stdout) + except JSONDecodeError: + raise HealthException(f"Invalid logcli response: {stdout}") + + outputter.show(data) + +def query_loki(from_dt, to_dt, since, query): + + loki_container_name = conf.global_config['loki']['loki_container_name'] + loki_port = conf.global_config['loki']['loki_port'] + loki_url = f"http://{loki_container_name}:{loki_port}" + + network_name = conf.global_config['podman']['network_name'] + logcli_container_name = conf.global_config['logcli']['logcli_container_name'] + + logcli_image_name = conf.global_config['logcli']['logcli_image_name'] + + podman_args = [ + "run", + "--rm", + "--replace", + "--network", + 'health-check-network', + "--name", + f"{logcli_container_name}", + f"{logcli_image_name}", + "query", + "--quiet", + "--output=jsonl", + f"--addr={loki_url}", + "--limit=150", + ] + + if to_dt and not from_dt: + # Doesn't make sense to have to without from + raise HealthException + + if from_dt: + podman_args.extend([ + f"--from={from_dt}", + ]) + + if to_dt: + # to_dt has priority over since if the two parameters are present + podman_args.extend([ + f"--to={to_dt}" + ]) + + else: + # Since should always have a default value + podman_args.extend([ + f"--since={since}" + ]) + else: + # The default "from" is "since" days ago. Since should always have a default value. + from_time = (datetime.utcnow() - timedelta(days=since)).isoformat() + podman_args.append(query) + stdout, stderr, _ = podman(cmd=podman_args) + return [stdout, stderr] + diff --git a/health-check/src/uyuni_health_check/main.py b/health-check/src/uyuni_health_check/main.py index 54dcbf6323b..4b6ac0a858e 100644 --- a/health-check/src/uyuni_health_check/main.py +++ b/health-check/src/uyuni_health_check/main.py @@ -10,7 +10,7 @@ wait_loki_init, download_component_build_image, ) -from uyuni_health_check.loki.logs_gatherer import show_full_error_logs +from uyuni_health_check.loki.logs_gatherer import show_full_error_logs, show_error_logs_stats from uyuni_health_check.config_loader import ConfigLoader from uyuni_health_check.exporters.supportconfig import exporter import uyuni_health_check.containers.manager @@ -60,8 +60,16 @@ def cli(ctx, supportconfig_path, verbose): type=int, help="Show logs from last X days. (Default: 7)", ) +@click.option( + "--from_datetime", + help="Start looking for logs at this absolute time ", +) +@click.option( + "--to_datetime", + help="Stop looking for logs at this absolute time", +) @click.pass_context -def run(ctx, logs, since): +def run(ctx, logs, from_datetime, to_datetime, since): """ Start execution of Uyuni Health Check @@ -114,14 +122,14 @@ def run(ctx, logs, since): prepare_grafana(verbose=verbose) console.print(Markdown("# Summary of metrics gatherered from Supportconfig")) - uyuni_health_check.metrics.show_supportconfig_metrics(metrics, console) + #uyuni_health_check.metrics.show_supportconfig_metrics(metrics, console) console.print(Markdown("## Relevant Errors")) - uyuni_health_check.metrics.show_error_logs_stats(since, console) + show_error_logs_stats(from_datetime, to_datetime, since, console) if logs: with console.pager(): - show_full_error_logs(since) + show_full_error_logs(from_datetime, to_datetime, since) console.print(Markdown("# Execution Finished")) diff --git a/health-check/src/uyuni_health_check/metrics.py b/health-check/src/uyuni_health_check/metrics.py index f20fb5f690d..321e65f1381 100644 --- a/health-check/src/uyuni_health_check/metrics.py +++ b/health-check/src/uyuni_health_check/metrics.py @@ -13,7 +13,7 @@ from rich.table import Table from rich.text import Text -from uyuni_health_check.utils import HealthException, run_command +from uyuni_health_check.utils import HealthException, run_command, console from uyuni_health_check.containers.manager import podman @@ -30,72 +30,6 @@ def show_supportconfig_metrics(metrics: dict, console: "Console"): justify="center", ) - -def show_error_logs_stats(since, console: "Console", loki=None): - """ - Get and show the error logs stats - """ - print( - Markdown(f"- Getting summary of errors in logs over the last {since} days...") - ) - print() - loki_url = loki or "http://uyuni_health_check_loki:3100" - process = podman( - [ - "run", - "-ti", - "--rm", - "--network", - "health-check-network", - "--name", - "uyuni_health_check_logcli", - "logcli", - "--quiet", - f"--addr={loki_url}", - "instant-query", - "--limit", - "150", - 'count_over_time({job=~".+"} |~ `(?i)error|(?i)severe|(?i)critical|(?i)fatal` [' - + str(since) - + "d])", - ], - ) - response = process.stdout.read() - try: - data = json.loads(response) - except JSONDecodeError: - raise HealthException(f"Invalid logcli response: {response}") - - if data: - console.print( - Panel( - Text( - f"Ooops! Errors found in the last {since} days.", - justify="center", - ) - ), - style="italic red blink", - ) - table = Table(show_header=True, header_style="bold magenta", expand=True) - table.add_column("File") - table.add_column("Errors") - - for metric in data: - table.add_row(metric["metric"]["filename"], metric["value"][1]) - - print(table) - else: - console.print( - Panel( - Text( - f"Good news! No errors detected in logs in the last {since} days.", - justify="center", - ) - ), - style="italic green", - ) - - def show_salt_jobs_summary(metrics: dict): table = Table(show_header=True, header_style="bold magenta") table.add_column("Salt function name") diff --git a/health-check/src/uyuni_health_check/outputter/outputter.py b/health-check/src/uyuni_health_check/outputter/outputter.py new file mode 100644 index 00000000000..1fa8fa16bf0 --- /dev/null +++ b/health-check/src/uyuni_health_check/outputter/outputter.py @@ -0,0 +1,36 @@ +from rich.panel import Panel +from rich.markdown import Markdown +from rich.panel import Panel +from rich.table import Table +from rich.text import Text +from uyuni_health_check.utils import console + +def show(data): + if data: + console.print( + Panel( + Text( + f"Ooops! Errors found!", + justify="center", + ) + ), + style="italic red blink", + ) + table = Table(show_header=True, header_style="bold magenta", expand=True) + table.add_column("File") + table.add_column("Errors") + + for metric in data: + table.add_row(metric["metric"]["filename"], metric["values"][-1][1]) + + console.print(table) + else: + console.print( + Panel( + Text( + f"Good news! No errors detected in logs.", + justify="center", + ) + ), + style="italic green", + ) \ No newline at end of file diff --git a/health-check/src/uyuni_health_check/utils.py b/health-check/src/uyuni_health_check/utils.py index dd21d6dd43c..f4f71cf25d3 100644 --- a/health-check/src/uyuni_health_check/utils.py +++ b/health-check/src/uyuni_health_check/utils.py @@ -5,11 +5,13 @@ console = Console() - def run_command(cmd, console=None, quiet=True, use_print=False): """ Runs a command """ + console = Console() + print("cmd parameter: ", cmd ) + print(" Running: " + ' '.join(cmd)) process = subprocess.Popen( cmd, stdout=subprocess.PIPE, @@ -18,17 +20,23 @@ def run_command(cmd, console=None, quiet=True, use_print=False): universal_newlines=True, ) + stdout, stderr = process.communicate() + if console and not quiet: - while True: - line = process.stdout.readline() or process.stderr.readline() - if not line: - break + for line in stdout.splitlines(): if use_print: console.print_json(line.strip()) else: console.log(Text.from_ansi(line.strip())) + + for line in stderr.splitlines(): + if use_print: + console.print_json(line.strip()) + else: + console.log(Text.from_ansi(line.strip())) + + returncode = process.returncode - returncode = process.wait() if returncode == 127: raise OSError(f"Command not found: {cmd[0]}") elif returncode == 125: @@ -37,9 +45,15 @@ def run_command(cmd, console=None, quiet=True, use_print=False): ) elif returncode == 255: raise HealthException(f"There has been an error running: {cmd}") - return process + + return [stdout, stderr, returncode] class HealthException(Exception): def __init__(self, message): super().__init__(message) + +if __name__ == "__main__": + print("hola") + cmd = ['podman', 'run', '--rm', '--replace', '--network', 'health-check-network', '--name', 'uyuni_health_check_logcli', 'logcli', 'query', '--quiet', '--output=jsonl', '--addr="http://uyuni_health_check_loki:3100"', '--from="2024-09-16T00:00:00Z"', '--to="2024-09-17T20:00:00Z"', '--limit=150', '\'{job="apache"}\''] + run_command(cmd, use_print=True) \ No newline at end of file