From e5c0f064fbb2ea1cf302119519e803d8f0d48eea Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 7 Sep 2022 06:48:33 -0400 Subject: [PATCH] Add basic code for commands validation --- kadi/commands/__init__.py | 5 + kadi/commands/commands_v2.py | 2 + kadi/commands/validate/__init__.py | 0 kadi/commands/validate/cmds_validate.py | 647 ++++++++++++++++++++ kadi/commands/validate/templates/index.html | 213 +++++++ 5 files changed, 867 insertions(+) create mode 100644 kadi/commands/validate/__init__.py create mode 100644 kadi/commands/validate/cmds_validate.py create mode 100644 kadi/commands/validate/templates/index.html diff --git a/kadi/commands/__init__.py b/kadi/commands/__init__.py index 3944a4e5..16fca5c1 100644 --- a/kadi/commands/__init__.py +++ b/kadi/commands/__init__.py @@ -49,6 +49,11 @@ class Conf(ConfigNamespace): "Google Sheet ID for command events (flight scenario).", ) + cmd_events_bad_times_gid = ConfigItem( + "1681877928", + "Google Sheet gid for bad times in command events (flight scenario)", + ) + star_id_match_halfwidth = ConfigItem( 1.5, "Half-width box size of star ID match for get_starcats() (arcsec)." ) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 2b126cb2..19e2fd67 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -47,6 +47,8 @@ APPROVED_LOADS_OCCWEB_DIR = Path("FOT/mission_planning/PRODUCTS/APPR_LOADS") # URL to download google sheets `doc_id` +# See https://stackoverflow.com/questions/33713084 (original question). +# See also kadi.commands.cmds_validate for the long-form URL. CMD_EVENTS_SHEET_URL = ( "https://docs.google.com/spreadsheets/d/{doc_id}/export?format=csv" ) diff --git a/kadi/commands/validate/__init__.py b/kadi/commands/validate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/kadi/commands/validate/cmds_validate.py b/kadi/commands/validate/cmds_validate.py new file mode 100644 index 00000000..a71cfae8 --- /dev/null +++ b/kadi/commands/validate/cmds_validate.py @@ -0,0 +1,647 @@ +#!/usr/bin/env python + +""" +""" +import base64 +import functools +import io +import logging +import os +import time +from itertools import count +from pathlib import Path + +import jinja2 +import matplotlib +import numpy as np +import requests +from astropy.table import Table, vstack + +import kadi +from kadi.commands import conf + +matplotlib.use("Agg") +import cheta.fetch_eng as fetch +import matplotlib.pyplot as plt +import Ska.Matplotlib +import Ska.Numpy +import Ska.Shell +from cheta.utils import logical_intervals +from cxotime import CxoTime +from Quaternion import Quat, normalize +from Ska.Matplotlib import cxctime2plotdate as cxc2pd +from Ska.Matplotlib import plot_cxctime + +from kadi.commands.states import interpolate_states + +kadi.logger.logLevel = "DEBUG" +# os.environ["KADI"] = os.path.expanduser("~/git/kadi/early") +os.environ["KADI_COMMANDS_VERSION"] = "2" +# os.environ["KADI_SCENARIO"] = "early" +# fetch.add_logging_handler() + +# URL to download Bad Times google sheet +# See https://stackoverflow.com/questions/33713084 (2nd answer) +BAD_TIMES_SHEET_URL = ( + "https://docs.google.com/spreadsheets/d/{doc_id}/export?" + "format=csv" + "&id={doc_id}" + "&gid={gid}" +) + +plot_cxctime = functools.partial(plot_cxctime, interactive=False) + + +plt.rcParams["axes.formatter.limits"] = (-4, 4) +plt.rcParams["font.size"] = 9 +TASK = "validate_states" +VERSION = 5 +TASK_DATA = os.path.join(os.environ["SKA"], "data", TASK) +URL = "http://cxc.harvard.edu/mta/ASPECT/" + TASK +logger = logging.getLogger(TASK) + +TITLE = { + "dp_pitch": "Pitch", + "obsid": "OBSID", + "tscpos": "TSCPOS (SIM-Z)", + "pcad_mode": "PCAD MODE", + "dither": "DITHER", + "letg": "LETG", + "hetg": "HETG", + "pointing": "Commanded ATT Radial Offset", + "roll": "Commanded ATT Roll Offset", +} + +LABELS = { + "dp_pitch": "Pitch (degrees)", + "obsid": "OBSID", + "tscpos": "SIM-Z (steps)", + "pcad_mode": "PCAD MODE", + "dither": "Dither", + "letg": "LETG", + "hetg": "HETG", + "pointing": "Radial Offset (arcsec)", + "roll": "Roll Offset (arcsec)", +} + + +SCALES = {"dither": 1.0} + +FMTS = { + "dp_pitch": "%.3f", + "obsid": "%d", + "dither": "%d", + "hetg": "%d", + "letg": "%d", + "pcad_mode": "%d", + "tscpos": "%d", + "pointing": "%.2f", + "roll": "%.2f", +} + +MODE_SOURCE = {"pcad_mode": "aopcadmd", "dither": "aodithen"} + +MODE_MSIDS = { + "pcad_mode": ["NMAN", "NPNT", "NSUN", "NULL", "PWRF", "RMAN", "STBY"], + "dither": ["ENAB", "DISA"], + "hetg": ["INSE", "RETR"], + "letg": ["INSE", "RETR"], +} + +# validation limits +# 'msid' : (( quantile, absolute max value )) +# Note that the quantile needs to be in the set (1, 5, 16, 50, 84, 95, 99) +VALIDATION_LIMITS = { + "DP_PITCH": ( + (1, 7.0), + (99, 7.0), + (5, 0.5), + (95, 0.5), + ), + "POINTING": ( + (1, 0.05), + (99, 0.05), + ), + "ROLL": ( + (1, 0.05), + (99, 0.05), + ), + "TSCPOS": ( + (1, 2.0), + (99, 2.0), + ), +} + +# number of tolerated differences for string / discrete msids +# 'msid' : n differences before violation recorded +# this is scaled by the number of toggles or expected +# changes in the msid +VALIDATION_SCALE_COUNT = {"OBSID": 2, "HETG": 2, "LETG": 2, "PCAD_MODE": 2, "DITHER": 3} + + +def config_logging(outdir, verbose): + """Set up file and console logger. + See http://docs.python.org/library/logging.html#logging-to-multiple-destinations + """ + # Disable auto-configuration of root logger by adding a null handler. + # This prevents other modules (e.g. Chandra.cmd_states) from generating + # a streamhandler by just calling logging.info(..). + class NullHandler(logging.Handler): + def emit(self, record): + pass + + rootlogger = logging.getLogger() + rootlogger.addHandler(NullHandler()) + + loglevel = {0: logging.CRITICAL, 1: logging.INFO, 2: logging.DEBUG}.get( + verbose, logging.INFO + ) + + logger = logging.getLogger(TASK) + logger.setLevel(loglevel) + + formatter = logging.Formatter("%(message)s") + + console = logging.StreamHandler() + console.setFormatter(formatter) + logger.addHandler(console) + + filehandler = logging.FileHandler( + filename=os.path.join(outdir, "run.dat"), mode="w" + ) + filehandler.setFormatter(formatter) + logger.addHandler(filehandler) + + +def get_telem_values(tstart, msids, days=14, dt=32.8, name_map={}): + """ + Fetch last ``days`` of available ``msids`` telemetry values before + time ``tstart``. + + :param tstart: start time for telemetry (secs) + :param msids: fetch msids list + :param days: length of telemetry request before ``tstart`` + :param dt: sample time (secs) + :param name_map: dict mapping msid to recarray col name + :returns: np recarray of requested telemetry values from fetch + """ + tstart = CxoTime(tstart).secs + start = CxoTime(tstart - days * 86400).date + stop = CxoTime(tstart).date + logger.info("Fetching telemetry between %s and %s" % (start, stop)) + + # TODO: also use MAUDE telemetry to get recent data. But this needs some + # care to ensure the data are valid. + # with fetch.data_source("cxc", "maude"): + msidset = fetch.Msidset(msids, start, stop) + + start = max(x.times[0] for x in msidset.values()) + stop = min(x.times[-1] for x in msidset.values()) + msidset.interpolate(dt, start, stop) + + # Finished when we found at least 10 good records (5 mins) + if len(msidset.times) < 10: + raise ValueError( + "Found no telemetry within %d days of %s" % (days, str(tstart)) + ) + + outnames = ["date"] + [name_map.get(x, x) for x in msids] + out = np.rec.fromarrays( + [msidset.times] + [msidset[x].vals for x in msids], names=outnames + ) + return out + + +def get_bad_mask(tlm): + mask = np.zeros(len(tlm), dtype="bool") + bad_times = get_bad_times() + for interval in bad_times: + bad = (tlm["date"] >= CxoTime(interval["start"]).secs) & ( + tlm["date"] < CxoTime(interval["stop"]).secs + ) + mask[bad] = True + return mask + + +@functools.lru_cache(maxsize=1) +def get_bad_times() -> Table: + url = BAD_TIMES_SHEET_URL.format( + doc_id=conf.cmd_events_flight_id, gid=conf.cmd_events_bad_times_gid + ) + logger.info(f"Getting bad times from {url}") + req = requests.get(url, timeout=30) + if req.status_code != 200: + raise ValueError(f"Failed to get bad times sheet: {req.status_code}") + + bad_times = Table.read(req.text, format="csv") + return bad_times + + +def validate_cmd_states(days=4, run_start_time=None, scenario=None): + # Store info relevant to processing for use in outputs + proc = dict( + run_user=os.environ["USER"], + run_time=time.ctime(), + errors=[], + ) + + tnow = CxoTime(run_start_time).secs + tstart = tnow + + # Get temperature telemetry for 3 weeks prior to min(tstart, NOW) + tlm = get_telem_values( + tstart, + [ + "3tscpos", + "dp_pitch", + "aoacaseq", + "aodithen", + "cacalsta", + "cobsrqid", + "aofunlst", + "aopcadmd", + "4ootgsel", + "4ootgmtn", + "aocmdqt1", + "aocmdqt2", + "aocmdqt3", + # "1de28avo", + # "1deicacu", + # "1dp28avo", + # "1dpicacu", + # "1dp28bvo", + # "1dpicbcu", + ], + days=days, + name_map={"3tscpos": "tscpos", "cobsrqid": "obsid"}, + ) + + states = get_states(tlm[0].date, tlm[-1].date) + + # Get bad time intervals + bad_time_mask = get_bad_mask(tlm) + + # Interpolate states onto the tlm.date grid + state_vals = interpolate_states(states, tlm["date"]) + + # "Forgive" dither intervals with dark current replicas + # This will also exclude dither disables that are in cmd states for standard dark cals + dark_mask = np.zeros(len(tlm), dtype="bool") + dark_times = [] + # Find dither "disable" states from tlm + dith_disa_states = logical_intervals(tlm["date"], tlm["aodithen"] == "DISA") + for state in dith_disa_states: + # Index back into telemetry for each of these constant dither disable states + idx0 = np.searchsorted(tlm["date"], state["tstart"], side="left") + idx1 = np.searchsorted(tlm["date"], state["tstop"], side="right") + # If any samples have aca calibration flag, mark interval for exclusion. + if np.any(tlm["cacalsta"][idx0:idx1] != "OFF "): + dark_mask[idx0:idx1] = True + dark_times.append({"start": state["datestart"], "stop": state["datestop"]}) + + # Calculate the 4th term of the commanded quaternions + cmd_q4 = np.sqrt( + np.abs(1.0 - tlm["aocmdqt1"] ** 2 - tlm["aocmdqt2"] ** 2 - tlm["aocmdqt3"] ** 2) + ) + raw_tlm_q = np.vstack( + [tlm["aocmdqt1"], tlm["aocmdqt2"], tlm["aocmdqt3"], cmd_q4] + ).transpose() + + # Calculate angle/roll differences in state cmd vs tlm cmd quaternions + raw_state_q = np.vstack( + [state_vals[n] for n in ["q1", "q2", "q3", "q4"]] + ).transpose() + tlm_q = normalize(raw_tlm_q) + # only use values that aren't NaNs + good = ~np.isnan(np.sum(tlm_q, axis=-1)) + # and are in NPNT + npnt = tlm["aopcadmd"] == "NPNT" + # and are in KALM after the first 2 sample of the transition + not_kalm = tlm["aoacaseq"] != "KALM" + kalm = ~(not_kalm | np.hstack([[False, False], not_kalm[:-2]])) + # and aren't during momentum unloads or in the first 2 samples after unloads + unload = tlm["aofunlst"] != "NONE" + no_unload = ~(unload | np.hstack([[False, False], unload[:-2]])) + ok = good & npnt & kalm & no_unload & ~bad_time_mask + state_q = normalize(raw_state_q) + dot_q = np.sum(tlm_q[ok] * state_q[ok], axis=-1) + dot_q[dot_q > 1] = 1 + angle_diff = np.degrees(2 * np.arccos(dot_q)) + angle_diff = np.min([angle_diff, 360 - angle_diff], axis=0) + roll_diff = Quat(q=tlm_q[ok]).roll - Quat(q=state_q[ok]).roll + roll_diff = np.min([roll_diff, 360 - roll_diff], axis=0) + + for msid in MODE_SOURCE: + tlm_col = np.zeros(len(tlm)) + state_col = np.zeros(len(tlm)) + for mode, idx in zip(MODE_MSIDS[msid], count()): + tlm_col[tlm[MODE_SOURCE[msid]] == mode] = idx + state_col[state_vals[msid] == mode] = idx + tlm = Ska.Numpy.add_column(tlm, msid, tlm_col) + state_vals = Ska.Numpy.add_column(state_vals, "{}_pred".format(msid), state_col) + + for msid in ["letg", "hetg"]: + txt = np.repeat("RETR", len(tlm)) + # use a combination of the select telemetry and the insertion telem to + # approximate the state_vals values + txt[(tlm["4ootgsel"] == msid.upper()) & (tlm["4ootgmtn"] == "INSE")] = "INSE" + tlm_col = np.zeros(len(tlm)) + state_col = np.zeros(len(tlm)) + for mode, idx in zip(MODE_MSIDS[msid], count()): + tlm_col[txt == mode] = idx + state_col[state_vals[msid] == mode] = idx + tlm = Ska.Numpy.add_column(tlm, msid, tlm_col) + state_vals = Ska.Numpy.add_column(state_vals, "{}_pred".format(msid), state_col) + + diff_only = { + "pointing": {"diff": angle_diff * 3600, "date": tlm["date"][ok]}, + "roll": {"diff": roll_diff * 3600, "date": tlm["date"][ok]}, + } + + pred = { + "dp_pitch": state_vals.pitch, + "obsid": state_vals.obsid, + "dither": state_vals["dither_pred"], + "pcad_mode": state_vals["pcad_mode_pred"], + "letg": state_vals["letg_pred"], + "hetg": state_vals["hetg_pred"], + "tscpos": state_vals.simpos, + "pointing": 1, + "roll": 1, + } + + plots_validation = [] + valid_viols = [] + logger.info("Making validation plots and quantile table") + quantiles = (1, 5, 16, 50, 84, 95, 99) + # store lines of quantile table in a string and write out later + quant_table = "" + quant_head = ",".join(["MSID"] + ["quant%d" % x for x in quantiles]) + quant_table += quant_head + "\n" + for fig_id, msid in enumerate(sorted(pred)): + plot = dict(msid=msid.upper()) + scale = SCALES.get(msid, 1.0) + fig, ax = plt.subplots(figsize=(7, 3.5)) + + if msid not in diff_only: + if msid in MODE_MSIDS: + state_msid = np.zeros(len(tlm)) + for mode, idx in zip(MODE_MSIDS[msid], count()): + state_msid[state_vals[msid] == mode] = idx + ticklocs, fig, ax = plot_cxctime( + tlm["date"], tlm[msid], fig=fig, ax=ax, fmt="-r" + ) + ticklocs, _, _ = plot_cxctime( + tlm["date"], state_msid, fig=fig, ax=ax, fmt="-b" + ) + ax.set_yticks(range(len(MODE_MSIDS[msid])), MODE_MSIDS[msid]) + else: + ticklocs, fig, ax = plot_cxctime( + tlm["date"], tlm[msid] / scale, fig=fig, ax=ax, fmt="-r" + ) + ticklocs, fig, ax = plot_cxctime( + tlm["date"], pred[msid] / scale, fig=fig, ax=ax, fmt="-b" + ) + else: + ticklocs, fig, ax = plot_cxctime( + diff_only[msid]["date"], + diff_only[msid]["diff"] / scale, + fig=fig, + ax=ax, + fmt="-k", + ) + plot["diff_only"] = msid in diff_only + ax.set_title(TITLE[msid]) + ax.set_ylabel(LABELS[msid]) + xlims = ax.get_xlim() + ylims = ax.get_ylim() + + bad_times = get_bad_times() + + # Add the time intervals of dark current calibrations that have been + # excluded from the diffs to the "bad_times" for validation so they also + # can be marked with grey rectangles in the plot. This is only really + # visible with interactive/zoomed plot. + if msid in ["dither", "pcad_mode"] and len(dark_times) > 0: + bad_times = vstack([bad_times, Table(dark_times)]) + + # Add "background" grey rectangles for excluded time regions to vs-time plot + for bad in bad_times: + bad_start = cxc2pd([CxoTime(bad["start"]).secs])[0] + bad_stop = cxc2pd([CxoTime(bad["stop"]).secs])[0] + if not ((bad_stop >= xlims[0]) & (bad_start <= xlims[1])): + continue + rect = matplotlib.patches.Rectangle( + (bad_start, ylims[0]), + bad_stop - bad_start, + ylims[1] - ylims[0], + alpha=0.2, + facecolor="black", + edgecolor="none", + ) + ax.add_patch(rect) + + ax.margins(0.05) + fig.tight_layout() + + out = io.BytesIO() + fig.savefig(out, format="JPEG") + plt.close(fig) + plot["lines"] = base64.b64encode(out.getvalue()).decode("ascii") + + if msid not in diff_only: + ok = ~bad_time_mask + if msid in ["dither", "pcad_mode"]: + # For these two validations also ignore intervals during a dark + # current calibration + ok &= ~dark_mask + diff = tlm[msid][ok] - pred[msid][ok] + else: + diff = diff_only[msid]["diff"] + + # Sort the diffs in-place because we're just using them in aggregate + diff = np.sort(diff) + + # if there are only a few residuals, don't bother with histograms + if msid.upper() in VALIDATION_SCALE_COUNT: + plot["samples"] = len(diff) + plot["diff_count"] = np.count_nonzero(diff) + plot["n_changes"] = 1 + np.count_nonzero(pred[msid][1:] - pred[msid][0:-1]) + if plot["diff_count"] < ( + plot["n_changes"] * VALIDATION_SCALE_COUNT[msid.upper()] + ): + plots_validation.append(plot) + continue + # if the msid exceeds the diff count, add a validation violation + else: + viol = { + "msid": "{}_diff_count".format(msid), + "value": plot["diff_count"], + "limit": plot["n_changes"] * VALIDATION_SCALE_COUNT[msid.upper()], + "quant": None, + } + valid_viols.append(viol) + logger.info( + "WARNING: %s %d discrete diffs exceed limit of %d" + % ( + msid, + plot["diff_count"], + plot["n_changes"] * VALIDATION_SCALE_COUNT[msid.upper()], + ) + ) + + # Make quantiles + if msid != "obsid": + quant_line = "%s" % msid + for quant in quantiles: + quant_val = diff[(len(diff) * quant) // 100] + plot["quant%02d" % quant] = FMTS[msid] % quant_val + quant_line += "," + FMTS[msid] % quant_val + quant_table += quant_line + "\n" + + for histscale in ("lin", "log"): + fig, ax = plt.subplots(figsize=(4, 3)) + ax.hist(diff / scale, bins=50, log=(histscale == "log")) + ax.set_title(msid.upper() + " residuals: telem - cmd states", fontsize=11) + ax.set_xlabel(LABELS[msid]) + fig.subplots_adjust(bottom=0.18) + fig.tight_layout() + + out = io.BytesIO() + fig.savefig(out, format="JPEG") + plt.close(fig) + plot["hist" + histscale] = base64.b64encode(out.getvalue()).decode("ascii") + + plots_validation.append(plot) + + valid_viols.extend(make_validation_viols(plots_validation)) + html = get_index_html(proc, plots_validation, valid_viols) + return html + + +def get_index_html(proc, plots_validation, valid_viols=None): + """ + Make output text in HTML format in outdir. + """ + context = { + "valid_viols": valid_viols, + "proc": proc, + "plots_validation": plots_validation, + } + index_template_file = Path(__file__).parent / "templates" / "index.html" + index_template = index_template_file.read_text() + # index_template = re.sub(r' %}\n', ' %}', index_template) + template = jinja2.Template(index_template) + out = template.render(context) + return out + + +def get_states_kadi(datestart, datestop): + # Local import for speed and for namespace clarity + from kadi.commands.states import get_states + + # from kadi.commands import conf + # TODO use v2 commands + + logger.info("Using kadi.commands.states to get cmd_states") + logger.info("Getting commanded states between %s - %s" % (datestart, datestop)) + + states = get_states(datestart, datestop) + states["tstart"] = CxoTime(states["datestart"]).secs + states["tstop"] = CxoTime(states["datestop"]).secs + + return states + + +def get_states(datestart, datestop): + """Get states exactly covering date range + + :param datestart: start date + :param datestop: stop date + :param db: database handle + :returns: np recarry of states + """ + states = get_states_kadi(datestart, datestop) + + # Set start and end state date/times to match telemetry span. Extend the + # state durations by a small amount because of a precision issue converting + # to date and back to secs. (The reference tstop could be just over the + # 0.001 precision of date and thus cause an out-of-bounds error when + # interpolating state values). + states[0]["tstart"] = CxoTime(datestart).secs - 0.01 + states[0]["datestart"] = CxoTime(states[0]["tstart"]).date + states[-1]["tstop"] = CxoTime(datestop).secs + 0.01 + states[-1]["datestop"] = CxoTime(states[-1]["tstop"]).date + + return states + + +def make_validation_viols(plots_validation): + """ + Find limit violations where MSID quantile values are outside the + allowed range. + """ + + logger.info("Checking for validation violations") + + viols = [] + + for plot in plots_validation: + # 'plot' is actually a structure with plot info and stats about the + # plotted data for a particular MSID. 'msid' can be a real MSID + # (1PDEAAT) or pseudo like 'POWER' + msid = plot["msid"] + + # Make sure validation limits exist for this MSID + if msid not in VALIDATION_LIMITS: + continue + + # Cycle through defined quantiles (e.g. 99 for 99%) and corresponding + # limit values for this MSID. + for quantile, limit in VALIDATION_LIMITS[msid]: + # Get the quantile statistic as calculated when making plots + msid_quantile_value = float(plot["quant%02d" % quantile]) + + # Check for a violation and take appropriate action + if abs(msid_quantile_value) > limit: + viol = { + "msid": msid, + "value": msid_quantile_value, + "limit": limit, + "quant": quantile, + } + viols.append(viol) + logger.info( + "WARNING: %s %d%% quantile value of %s exceeds limit of %.2f" + % (msid, quantile, msid_quantile_value, limit) + ) + + return viols + + +def get_options(): + from optparse import OptionParser + + parser = OptionParser() + parser.set_defaults() + parser.add_option( + "--days", type="float", default=4.0, help="Days of validation data (days)" + ) + parser.add_option( + "--run-start-time", help="Mock tool run time for regression testing" + ) + parser.add_option( + "--verbose", + type="int", + default=2, + help="Verbosity (0=quiet, 1=normal, 2=debug)", + ) + + opt, args = parser.parse_args() + return opt, args + + +if __name__ == "__main__": + opt, args = get_options() + out = validate_cmd_states(days=opt.days, run_start_time=opt.run_start_time) + Path("index.html").write_text(out) diff --git a/kadi/commands/validate/templates/index.html b/kadi/commands/validate/templates/index.html new file mode 100644 index 00000000..b996f21d --- /dev/null +++ b/kadi/commands/validate/templates/index.html @@ -0,0 +1,213 @@ + + + + + + + index.rst + + + + +
+ +
+

Commanded States Database Verification

+
+

Summary

+ + + + + + + + + + + + + + + + +
Run time{{proc.run_time}} by {{proc.run_user}}
Run logrun.dat
Statesstates.dat
+
+
+
+

MSID Validation

+
+

MSID quantiles

+ + + + + + + + + + + + + + + + {% for plot in plots_validation %} + {% if plot.quant01 %} + + + + + + + + + + + {% endif %} + {% endfor%} + +
MSID1%5%16%50%84%95%99%
{{plot.msid}}{{plot.quant01}}{{plot.quant05}}{{plot.quant16}}{{plot.quant50}}{{plot.quant84}}{{plot.quant95}}{{plot.quant99}}
+
+ + {% if valid_viols %} +
+

Validation Violations

+ + + + + + + + + + + + + + {% for viol in valid_viols %} + + + + + + + {% endfor%} + + + {% else %} +

No Validation Violations

+ {% endif %} + + {% for plot in plots_validation %} +
+

{{plot.msid}}

+ + {% if plot.diff_only %} +

Black = difference values

+ {% else %} +

Red = telemetry, blue = model

+ {% endif %} + + + + {% if plot.histlog %} + + {% else %} +

No histogram provided.

+ {% endif %} + + {% if plot.histlin %} + + {% endif %} + + {% if plot.samples %} +

{{ plot.diff_count }} non-identical samples of {{ plot.samples + }} samples.

+ {% endif %} +
+ {% endfor %} + + + + + + \ No newline at end of file
MSIDQuantileValueLimit
{{viol.msid}}{{viol.quantile}}{{viol.value}}{{viol.limit}}