From 08a0628eff8b86b401fb2348642db43e33212963 Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Tue, 13 Sep 2022 13:31:25 -0300 Subject: [PATCH] Added some basic preprocessing - Now you can parametrize the YAML config. Related to #233 and #243 --- CHANGELOG.md | 3 +++ README.md | 20 ++++++++++++++++- docs/README.in | 16 ++++++++++++++ kibot/__main__.py | 37 +++++++++++++++++++++++++------- kibot/config_reader.py | 12 +++++++++++ kibot/gs.py | 2 ++ tests/test_plot/test_position.py | 21 ++++++++++++++++++ 7 files changed, 102 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0724df67..892a04ecb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- General things: + - Some basic preprocessing, now you can parametrize the YAML config. + (See #233 #243) - New outputs: - PCB_Variant: saves a PCB with filters and variants applied. - File_Copy: used to copy files to the output directory. (#279) diff --git a/README.md b/README.md index dfd924758..22280d933 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ * [Importing outputs from another file](#importing-outputs-from-another-file) * [Using other output as base for a new one](#using-other-output-as-base-for-a-new-one) * [Importing filters and variants from another file](#importing-filters-and-variants-from-another-file) + * [Doing YAML substitution or preprocessing](#doing-yaml-substitution-or-preprocessing) * [Usage](#usage) * [Usage for CI/CD](#usage-for-cicd) * [Github Actions](#usage-of-github-actions) @@ -3423,6 +3424,21 @@ import: This will import all outputs and filters, but not variants or globals. Also note that imported globals has more precedence than the ones defined in the same file. +### Doing YAML substitution or preprocessing + +Sometimes you could want to change values in the YAML depending on external stuff, +or just want to be able to change something for each variant run. + +In this case you can use external tools to create various YAML files using a template, +but you can also use KiBot's definitions. + +The definitions allows you to replace tags like `@VARIABLE@` by some specified value. +These definitions can be specified at the command line using the `-E` option. +As an example: `-E UNITS=millimeters` will replace all `@UNITS@` markers by `millimeters`. +This is applied to all YAML files loaded, so this propagates to all the imported YAML files. + +You can use `-E` as many times as you need. + ## Usage For a quick start just go to the project's dir and run: @@ -3526,7 +3542,8 @@ KiBot: KiCad automation tool for documents generation Usage: kibot [-b BOARD] [-e SCHEMA] [-c CONFIG] [-d OUT_DIR] [-s PRE] [-D] - [-q | -v...] [-C | -i | -n] [-m MKFILE] [-A] [-g DEF] ... [TARGET...] + [-q | -v...] [-C | -i | -n] [-m MKFILE] [-A] [-g DEF] ... + [-E DEF] ... [TARGET...] kibot [-v...] [-b BOARD] [-e SCHEMA] [-c PLOT_CONFIG] --list kibot [-v...] [-b BOARD] [-d OUT_DIR] [-p | -P] --example kibot [-v...] [--start PATH] [-d OUT_DIR] [--dry] [-t, --type TYPE]... @@ -3552,6 +3569,7 @@ Options: -d OUT_DIR, --out-dir OUT_DIR The output directory [default: .] -D, --dont-stop Try to continue if an output fails -e SCHEMA, --schematic SCHEMA The schematic file (.sch) + -E DEF, --define DEF Define preprocessor value (VAR=VAL) -g DEF, --global-redef DEF Overwrite a global value (VAR=VAL) -i, --invert-sel Generate the outputs not listed as targets -l, --list List available outputs (in the config file) diff --git a/docs/README.in b/docs/README.in index 333b95868..ffa392367 100644 --- a/docs/README.in +++ b/docs/README.in @@ -59,6 +59,7 @@ * [Importing outputs from another file](#importing-outputs-from-another-file) * [Using other output as base for a new one](#using-other-output-as-base-for-a-new-one) * [Importing filters and variants from another file](#importing-filters-and-variants-from-another-file) + * [Doing YAML substitution or preprocessing](#doing-yaml-substitution-or-preprocessing) * [Usage](#usage) * [Usage for CI/CD](#usage-for-cicd) * [Github Actions](#usage-of-github-actions) @@ -1195,6 +1196,21 @@ import: This will import all outputs and filters, but not variants or globals. Also note that imported globals has more precedence than the ones defined in the same file. +### Doing YAML substitution or preprocessing + +Sometimes you could want to change values in the YAML depending on external stuff, +or just want to be able to change something for each variant run. + +In this case you can use external tools to create various YAML files using a template, +but you can also use KiBot's definitions. + +The definitions allows you to replace tags like `@VARIABLE@` by some specified value. +These definitions can be specified at the command line using the `-E` option. +As an example: `-E UNITS=millimeters` will replace all `@UNITS@` markers by `millimeters`. +This is applied to all YAML files loaded, so this propagates to all the imported YAML files. + +You can use `-E` as many times as you need. + ## Usage For a quick start just go to the project's dir and run: diff --git a/kibot/__main__.py b/kibot/__main__.py index fbc1efc5a..e135d5220 100644 --- a/kibot/__main__.py +++ b/kibot/__main__.py @@ -9,7 +9,8 @@ Usage: kibot [-b BOARD] [-e SCHEMA] [-c CONFIG] [-d OUT_DIR] [-s PRE] [-D] - [-q | -v...] [-C | -i | -n] [-m MKFILE] [-A] [-g DEF] ... [TARGET...] + [-q | -v...] [-C | -i | -n] [-m MKFILE] [-A] [-g DEF] ... + [-E DEF] ... [TARGET...] kibot [-v...] [-b BOARD] [-e SCHEMA] [-c PLOT_CONFIG] --list kibot [-v...] [-b BOARD] [-d OUT_DIR] [-p | -P] --example kibot [-v...] [--start PATH] [-d OUT_DIR] [--dry] [-t, --type TYPE]... @@ -35,6 +36,7 @@ -d OUT_DIR, --out-dir OUT_DIR The output directory [default: .] -D, --dont-stop Try to continue if an output fails -e SCHEMA, --schematic SCHEMA The schematic file (.sch) + -E DEF, --define DEF Define preprocessor value (VAR=VAL) -g DEF, --global-redef DEF Overwrite a global value (VAR=VAL) -i, --invert-sel Generate the outputs not listed as targets -l, --list List available outputs (in the config file) @@ -240,6 +242,24 @@ def detect_kicad(): logger.debug('KiCad config path {}'.format(GS.kicad_conf_path)) +def parse_defines(args): + for define in args.define: + if '=' not in define: + logger.error('Malformed `define` option, must be VARIABLE=VALUE ({})'.format(define)) + sys.exit(EXIT_BAD_ARGS) + var = define.split('=')[0] + GS.cli_defines[var] = define[len(var)+1:] + + +def parse_global_redef(args): + for redef in args.global_redef: + if '=' not in redef: + logger.error('Malformed global-redef option, must be VARIABLE=VALUE ({})'.format(redef)) + sys.exit(EXIT_BAD_ARGS) + var = redef.split('=')[0] + GS.cli_global_defs[var] = redef[len(var)+1:] + + def main(): set_locale() ver = 'KiBot '+__version__+' - '+__copyright__+' - License: '+__license__ @@ -257,12 +277,7 @@ def main(): os.environ['INTERACTIVE_HTML_BOM_NO_DISPLAY'] = 'True' # Parse global overwrite options - for redef in args.global_redef: - if '=' not in redef: - logger.error('Malformed global-redef option, must be VARIABLE=VALUE ({})'.format(redef)) - sys.exit(EXIT_BAD_ARGS) - var = redef.split('=')[0] - GS.cli_global_defs[var] = redef[len(var)+1:] + parse_global_redef(args) # Disable auto-download if needed if args.no_auto_download: @@ -313,13 +328,19 @@ def main(): # Determine the project file GS.set_pro(solve_project_file()) + # Parse preprocessor defines + parse_defines(args) + # Read the config file cr = CfgYamlReader() outputs = None try: # The Python way ... with gzip.open(plot_config) as cf_file: - outputs = cr.read(cf_file) + try: + outputs = cr.read(cf_file) + except KiPlotConfigurationError as e: + config_error(str(e)) except OSError: pass if outputs is None: diff --git a/kibot/config_reader.py b/kibot/config_reader.py index e43552188..1b390ad2f 100644 --- a/kibot/config_reader.py +++ b/kibot/config_reader.py @@ -9,6 +9,7 @@ Class to read KiBot config files """ +import io import os import json from sys import (exit, maxsize) @@ -446,6 +447,17 @@ def _parse_import(self, imp, name, apply=True, depth=0): return all_collected def load_yaml(self, fstream): + if GS.cli_defines: + # Load the file to memory so we can preprocess it + content = fstream.read() + logger.debug('Applying preprocessor definitions') + # Replace all + for k, v in GS.cli_defines.items(): + key = '@'+k+'@' + logger.debugl(2, '- Replacing {} -> {}'.format(key, v)) + content = content.replace(key, v) + # Create an stream from the string + fstream = io.StringIO(content) try: data = yaml.safe_load(fstream) except yaml.YAMLError as e: diff --git a/kibot/gs.py b/kibot/gs.py index 4367ad358..ec6667f19 100644 --- a/kibot/gs.py +++ b/kibot/gs.py @@ -86,6 +86,8 @@ class GS(object): test_boolean = True test_number = 5 stackup = None + # Preprocessor definitions + cli_defines = {} # # Global defaults # diff --git a/tests/test_plot/test_position.py b/tests/test_plot/test_position.py index 59df90b23..2eb6ff5ab 100644 --- a/tests/test_plot/test_position.py +++ b/tests/test_plot/test_position.py @@ -143,6 +143,27 @@ def test_position_csv_cols(test_dir): ctx.clean_up() +def test_position_3Rs_pre_csv(test_dir): + """ Test using preprocessor """ + ctx = context.TestContext(test_dir, '3Rs', 'simple_position_csv_pre', POS_DIR+'_millimeters') + ctx.run(extra=['-E', 'UNITS=millimeters']) + pos_top = ctx.get_pos_top_csv_filename() + pos_bot = ctx.get_pos_bot_csv_filename() + ctx.expect_out_file(pos_top) + ctx.expect_out_file(pos_bot) + expect_position(ctx, pos_top, ['R1'], ['R2', 'R3'], csv=True) + expect_position(ctx, pos_bot, ['R2'], ['R1', 'R3'], csv=True) + ctx.sub_dir = POS_DIR+'_inches' + ctx.run(extra=['-E', 'UNITS=inches']) + pos_top = ctx.get_pos_top_csv_filename() + pos_bot = ctx.get_pos_bot_csv_filename() + ctx.expect_out_file(pos_top) + ctx.expect_out_file(pos_bot) + expect_position(ctx, pos_top, ['R1'], ['R2', 'R3'], csv=True, inches=True) + expect_position(ctx, pos_bot, ['R2'], ['R1', 'R3'], csv=True, inches=True) + ctx.clean_up() + + def test_position_3Rs_unified_csv(test_dir): """ Also test the quiet mode """ ctx = context.TestContext(test_dir, '3Rs', 'simple_position_unified_csv', POS_DIR)