From 750598918e03c44b48e725e737b181f031ca2fd9 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 15 Jun 2020 16:51:15 +0000 Subject: [PATCH 01/13] wip - autodoc confdefs --- synapse/lib/config.py | 23 +++++++++++--- synapse/tools/autodoc.py | 69 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/synapse/lib/config.py b/synapse/lib/config.py index e07ad46a99..4a1ab2fc4d 100644 --- a/synapse/lib/config.py +++ b/synapse/lib/config.py @@ -234,11 +234,8 @@ def setConfFromEnvs(self): Returns: None: Returns None. ''' - for name, conf in self.json_schema.get('properties', {}).items(): - if conf.get('hideconf'): - continue - - envar = make_envar_name(name, prefix=self.envar_prefix) + name2envar = self.getEnvarMapping() + for name, envar in name2envar.items(): envv = os.getenv(envar) if envv is not None: envv = yaml.safe_load(envv) @@ -246,6 +243,22 @@ def setConfFromEnvs(self): if resp == envv: logger.debug(f'Set config valu from envar: [{envar}]') + def getEnvarMapping(self): + ''' + Get a mapping of config values to envars. + + Configuration values which have the ``hideconf`` value set to True are not resolved from environment + variables. + ''' + ret = {} + for name, conf in self.json_schema.get('properties', {}).items(): + if conf.get('hideconf'): + continue + + envar = make_envar_name(name, prefix=self.envar_prefix) + ret[name] = envar + return ret + # General methods def reqConfValid(self): ''' diff --git a/synapse/tools/autodoc.py b/synapse/tools/autodoc.py index 4734463f0e..4f0626f0f6 100644 --- a/synapse/tools/autodoc.py +++ b/synapse/tools/autodoc.py @@ -9,7 +9,9 @@ import synapse.cortex as s_cortex import synapse.telepath as s_telepath +import synapse.lib.config as s_config import synapse.lib.output as s_output +import synapse.lib.dyndeps as s_dyndeps logger = logging.getLogger(__name__) @@ -347,6 +349,54 @@ async def docModel(outp, # outp.printf(rst2.getRstText()) return rst, rst2 +async def doc_confdefcs(ctor): + cls = s_dyndeps.tryDynLocal(ctor) + + print(cls) + + if not hasattr(cls, 'confdefs'): + raise Exception('ctor must have a confdefs attr') + + rst = RstHelp() + print(cls.confdefs) + clsname = cls.__name__ + conf = cls.initCellConf() + print(conf) + + rst.addHead(f'{clsname} Configuration Options', lvl=0) + + # access raw config data + + # Get raw envars + name2envar = conf.getEnvarMapping() + + # Get argparse mappping? + + schema = conf.json_schema.get('properties', {}) + + for name, conf in sorted(schema.items(), key=lambda x: x[0]): + + nodesc = f'No description avilable for ``{name}``.' + print(name, conf) + hname = name + if ':' in name: + hname = name.replace(':', raw_back_slash_colon) + + rst.addHead(hname, lvl=1) + + desc = conf.get('description', nodesc) + lines = [] + lines.append(desc) + + envar = name2envar.get(name) + if envar: + lines.append(f'This option can be resolved via the environment variable: {envar}') + + rst.addLines(*lines) + + return rst, clsname + + async def main(argv, outp=None): if outp is None: @@ -372,6 +422,17 @@ async def main(argv, outp=None): with open(s_common.genpath(opts.savedir, 'datamodel_forms.rst'), 'wb') as fd: fd.write(rstforms.getRstText().encode()) + if opts.doc_cell: + confdocs, cname = await doc_confdefcs(opts.doc_cell) + + print(confdocs, cname) + + if opts.savedir: + with open(s_common.genpath(opts.savedir, f'confdefs_{cname}.rst'), 'wb') as fd: + fd.write(confdocs.getRstText().encode()) + + print(confdocs.getRstText()) + return 0 def makeargparser(): @@ -382,8 +443,12 @@ def makeargparser(): help='Cortex URL for model inspection') pars.add_argument('--savedir', default=None, help='Save output to the given directory') - pars.add_argument('--doc-model', action='store_true', default=False, - help='Generate RST docs for the DataModel within a cortex') + doc_type = pars.add_mutually_exclusive_group() + doc_type.add_argument('--doc-model', action='store_true', default=False, + help='Generate RST docs for the DataModel within a cortex') + doc_type.add_argument('--doc-cell', default=None, + help='Generate RST docs for the Confdefs for a given Cell ctor') + return pars if __name__ == '__main__': # pragma: no cover From 3202d552c707f7f13bee7e6ea094d5937f1c21ce Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 15 Jun 2020 17:24:11 +0000 Subject: [PATCH 02/13] WIP --- docs/conf.py | 15 +++++++++++++++ docs/synapse/devops.rst | 9 +++++++++ synapse/tools/autodoc.py | 32 ++++++++++++++++++++++++++++---- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 2ff75d8518..10a7df1736 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -208,6 +208,20 @@ def run_modeldoc(_): '--savedir', './docs/synapse/autodocs'] subprocess.run(args, cwd=synpd) +def run_confdocs(_): + import synapse + import subprocess + abssynf = os.path.abspath(synapse.__file__) + synbd = os.path.split(abssynf)[0] # Split off __init__ + synpd = os.path.split(synbd)[0] # split off the synapse module directory + baseargs = ['python', '-m', 'synapse.tools.autodoc', '--savedir', + './docs/synapse/autodocs', '--doc-cell'] + ctors = ('synapse.axon.Axon', 'synapse.cortex.Cortex', 'synapse.cryotank.CryoCell') + for ctor in ctors: + args = baseargs.copy() + args.append(ctor) + subprocess.run(args, cwd=synpd) + def convert_ipynb(_): import synapse.common as s_common import nbconvert.nbconvertapp as nba @@ -228,4 +242,5 @@ def convert_ipynb(_): def setup(app): app.connect('builder-inited', run_apidoc) app.connect('builder-inited', run_modeldoc) + app.connect('builder-inited', run_confdocs) app.connect('builder-inited', convert_ipynb) diff --git a/docs/synapse/devops.rst b/docs/synapse/devops.rst index be76e29658..a46a9a161e 100644 --- a/docs/synapse/devops.rst +++ b/docs/synapse/devops.rst @@ -6,6 +6,15 @@ Synapse DevOps Guide #################### + +.. toctree:: + :titlesonly: + + autodocs/conf_axon + autodocs/conf_cortex + autodocs/conf_cryocell + + General ======= diff --git a/synapse/tools/autodoc.py b/synapse/tools/autodoc.py index 4f0626f0f6..d54b853bbc 100644 --- a/synapse/tools/autodoc.py +++ b/synapse/tools/autodoc.py @@ -364,19 +364,23 @@ async def doc_confdefcs(ctor): print(conf) rst.addHead(f'{clsname} Configuration Options', lvl=0) + rst.addLines(f'The following are boot-time configuration options for a {clsname}') + + # FIXME Make this a parameter so we can link from non-synapse stacks here. + rst.addLines(f'See :ref:`devops_cell_config` for details on how to set these options..') # access raw config data # Get raw envars name2envar = conf.getEnvarMapping() - + name2cmdline = {} # Get argparse mappping? schema = conf.json_schema.get('properties', {}) for name, conf in sorted(schema.items(), key=lambda x: x[0]): - nodesc = f'No description avilable for ``{name}``.' + nodesc = f'No description available for ``{name}``.' print(name, conf) hname = name if ':' in name: @@ -388,9 +392,29 @@ async def doc_confdefcs(ctor): lines = [] lines.append(desc) + # Type/additional information + + lines.append('\n') + # lines.append('Configuration properties:\n') + + ctyp = conf.get('type') + lines.append('Type') + lines.append(f' {ctyp}\n') + + defval = conf.get('default', s_common.novalu) + if defval is not s_common.novalu: + lines.append('Default Value') + lines.append(f' ``{repr(defval)}``\n') + envar = name2envar.get(name) if envar: - lines.append(f'This option can be resolved via the environment variable: {envar}') + lines.append('Environment Variable') + lines.append(f' {envar}\n') + + cmdline = name2cmdline.get(name) + if cmdline: + lines.append('Command Line Argument') + lines.append(f' --{cmdline}\n') rst.addLines(*lines) @@ -428,7 +452,7 @@ async def main(argv, outp=None): print(confdocs, cname) if opts.savedir: - with open(s_common.genpath(opts.savedir, f'confdefs_{cname}.rst'), 'wb') as fd: + with open(s_common.genpath(opts.savedir, f'conf_{cname.lower()}.rst'), 'wb') as fd: fd.write(confdocs.getRstText().encode()) print(confdocs.getRstText()) From 9d40adeb462e16cf1536421414c64036a2773b8f Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 15 Jun 2020 17:59:40 +0000 Subject: [PATCH 03/13] Embed command line argument. --- synapse/lib/config.py | 8 ++++++++ synapse/tools/autodoc.py | 14 +++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/synapse/lib/config.py b/synapse/lib/config.py index 4a1ab2fc4d..22ec2e5719 100644 --- a/synapse/lib/config.py +++ b/synapse/lib/config.py @@ -137,6 +137,7 @@ def __init__(self, conf = {} self.conf = conf self._argparse_conf_names = {} + self._argparse_conf_parsed_names = {} self.envar_prefix = envar_prefix self.validator = getJsValidator(self.json_schema) @@ -183,11 +184,18 @@ def getArgParseArgs(self): parsed_name = name.replace(':', '-') replace_name = name.replace(':', '_') self._argparse_conf_names[replace_name] = name + self._argparse_conf_parsed_names[name] = parsed_name argname = '--' + parsed_name argdata.append((argname, akwargs)) return argdata + def getCmdlineMapping(self): + if not self._argparse_conf_parsed_names: + # Giv a shot at populating the data + _ = self.getArgParseArgs() + return {k: v for k, v in self._argparse_conf_parsed_names.items()} + def setConfFromOpts(self, opts): ''' Set the opts for a conf object from a namespace object. diff --git a/synapse/tools/autodoc.py b/synapse/tools/autodoc.py index d54b853bbc..aee345683f 100644 --- a/synapse/tools/autodoc.py +++ b/synapse/tools/autodoc.py @@ -349,7 +349,7 @@ async def docModel(outp, # outp.printf(rst2.getRstText()) return rst, rst2 -async def doc_confdefcs(ctor): +async def docConfdefs(ctor, reflink=':ref:`devops_cell_config`'): cls = s_dyndeps.tryDynLocal(ctor) print(cls) @@ -360,7 +360,7 @@ async def doc_confdefcs(ctor): rst = RstHelp() print(cls.confdefs) clsname = cls.__name__ - conf = cls.initCellConf() + conf = cls.initCellConf() # type: s_config.Config print(conf) rst.addHead(f'{clsname} Configuration Options', lvl=0) @@ -373,7 +373,7 @@ async def doc_confdefcs(ctor): # Get raw envars name2envar = conf.getEnvarMapping() - name2cmdline = {} + name2cmdline = conf.getCmdlineMapping() # Get argparse mappping? schema = conf.json_schema.get('properties', {}) @@ -399,7 +399,7 @@ async def doc_confdefcs(ctor): ctyp = conf.get('type') lines.append('Type') - lines.append(f' {ctyp}\n') + lines.append(f' ``{ctyp}``\n') defval = conf.get('default', s_common.novalu) if defval is not s_common.novalu: @@ -409,12 +409,12 @@ async def doc_confdefcs(ctor): envar = name2envar.get(name) if envar: lines.append('Environment Variable') - lines.append(f' {envar}\n') + lines.append(f' ``{envar}``\n') cmdline = name2cmdline.get(name) if cmdline: lines.append('Command Line Argument') - lines.append(f' --{cmdline}\n') + lines.append(f' ``--{cmdline}``\n') rst.addLines(*lines) @@ -447,7 +447,7 @@ async def main(argv, outp=None): fd.write(rstforms.getRstText().encode()) if opts.doc_cell: - confdocs, cname = await doc_confdefcs(opts.doc_cell) + confdocs, cname = await docConfdefs(opts.doc_cell) print(confdocs, cname) From 50ef9c58d107d9a4a289bd09eeba31cb2f21083b Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 15 Jun 2020 18:21:19 +0000 Subject: [PATCH 04/13] Small fixes --- docs/synapse/devguides/backups.rst | 0 docs/synapse/devguides/devops_axon.rst | 4 + docs/synapse/devguides/devops_cortex.rst | 92 +++++++++ docs/synapse/devguides/devops_cryotank.rst | 4 + docs/synapse/devguides/devops_general.rst | 106 ++++++++++ docs/synapse/devops.rst | 214 +-------------------- synapse/tools/autodoc.py | 18 +- 7 files changed, 222 insertions(+), 216 deletions(-) create mode 100644 docs/synapse/devguides/backups.rst create mode 100644 docs/synapse/devguides/devops_axon.rst create mode 100644 docs/synapse/devguides/devops_cortex.rst create mode 100644 docs/synapse/devguides/devops_cryotank.rst create mode 100644 docs/synapse/devguides/devops_general.rst diff --git a/docs/synapse/devguides/backups.rst b/docs/synapse/devguides/backups.rst new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/synapse/devguides/devops_axon.rst b/docs/synapse/devguides/devops_axon.rst new file mode 100644 index 0000000000..6ff11d3b86 --- /dev/null +++ b/docs/synapse/devguides/devops_axon.rst @@ -0,0 +1,4 @@ +Axon Operations +=============== + +TBD diff --git a/docs/synapse/devguides/devops_cortex.rst b/docs/synapse/devguides/devops_cortex.rst new file mode 100644 index 0000000000..f754c5fb90 --- /dev/null +++ b/docs/synapse/devguides/devops_cortex.rst @@ -0,0 +1,92 @@ +Cortex Operations +================= + +Docker Deployment +----------------- + +Configuring A Mirror +-------------------- + +.. _200_migration: + +0.1.x to 2.x.x Migration +------------------------ + +Two tools have been created to execute migration of an ``0.1.x`` Cortex to ``2.x.x``: + +* ``migrate_200`` migrates all data from the source to a new destination ``2.x.x`` Cortex. +* ``sync_200`` allows for a backup to be migrated and then synchronized with a running Cortex to facilitate minimal downtime. + +Migration Quickstart +******************** + +.. note:: + + The duration of migration is proportional to the amount of data stored, and is highly dependent on + the available system resources (especially disk I/O). For larger Cortexes it is recommended to + run migration on hardware without other highly active processes. + +#. Update the source to the latest Synapse ``0.1.x`` release. +#. Create a backup of the ``0.1.x`` Cortex. +#. In a new location install Synapse ``2.x.x`` and copy any custom modules / configurations present in the ``0.1.x`` environment. +#. Start migration using the backup as the source:: + + python -m synapse.tools.migrate_200 --src --dest + +#. Inspect the migration output for any errors that may require action (see :ref:`migration-errors` for details). +#. Startup the ``2.x.x`` Cortex. + +Migration Options +***************** + +* ``--from-last`` restarts node migration from the last checkpoint (automatically saved at periodic intervals). +* ``--safety-off`` disables form value normalization checks as a pre-condition to migrate nodes (may allow migration to run faster). +* ``--src-dedicated`` opens the source layer slabs with locked memory (must have sufficient memory available). + +Additional options that, if specified, will *not* run a migration process: + +* ``--form-counts`` is a helper utility that scans all nodes and produces counts by form for source and destination. +* ``--dump-errors`` saves migration errors to a file in the migration directory in msgpack format. + +.. _migration-errors: + +Migration Errors +**************** + +During node migration the following errors may occur, all of which indicate that the node was not migrated: + +``Unable to determine stortype`` or ``Buid/norming exception: NoSuchForm`` + +* A custom form/property may not have been properly loaded into the ``2.x.x`` environment. +* The node may not have been properly updated in a prior ``0.1.x`` datamodel migration and therefore no longer exists. + +``Normed form val does not match inbound`` or ``Calculated buid does not match inbound`` + +* Is likely due to a node that was not properly re-normalized after a prior Synapse update. + +Post-migration Synchronization +****************************** + +After migration, the ``sync_200`` service can be used to push post-backup changes to the migrated ``2.x.x`` Cortex, +and keep it updated until cut-over. ``sync_200`` uses splices to translate the changes, and therefore they must +be enabled on the source Cortex. In order to control and monitor synchronization, ``sync_200`` can be added as a Storm service. + +When synchronization is started the service will enable "migration mode" on the destination ``2.x.x`` Cortex, which +prevents cron jobs and triggers from running. Migration mode will then be disabled when the synchronization is +stopped or when the Cortex is restarted. + +#. Complete migration, including starting up the ``2.x.x`` Cortex. +#. Locate the saved splice offset file from migration at ``/migration/lyroffs.yaml``. +#. Start the ``sync_200`` service (shown with the optional ``--auth-passwd`` to bootstrap the root user):: + + python -m synapse.tools.sync_200 \ + --auth-passwd secret --offsfile \ + --src <01x_telepath_url> --dest <20x_telepath_url> + +#. Add the Storm service to the Cortex and use the available commands to start synchronization. +#. When ready to cut-over, and the read status is up-to-date, stop the synchronization using the ``stopsync`` command. + +Cleanup +******* + +After migration is fully complete, delete the now-unused directory "migration" inside the cortex directory. diff --git a/docs/synapse/devguides/devops_cryotank.rst b/docs/synapse/devguides/devops_cryotank.rst new file mode 100644 index 0000000000..3632682943 --- /dev/null +++ b/docs/synapse/devguides/devops_cryotank.rst @@ -0,0 +1,4 @@ +Cryotank Operations +=================== + +TDB \ No newline at end of file diff --git a/docs/synapse/devguides/devops_general.rst b/docs/synapse/devguides/devops_general.rst new file mode 100644 index 0000000000..408b419494 --- /dev/null +++ b/docs/synapse/devguides/devops_general.rst @@ -0,0 +1,106 @@ +General Devops +============== + +Backups +------- + +It is strongly recommended that users schedule regular backups of all services deployed within their **Synapse** +ecosystem. Each service can be backed up using the **Synapse** backup tool: ``synapse.tools.backup``. + +The **Synapse** service architecture is designed to contain everything a service needs within the directory you +specify during service startup. Take, for example, a **Cortex** started with:: + python -m synapse.servers.cortex /data/cortex00 + +The **Cortex** will be completely contained within the service working directory ``/data/cortex00``. The synapse tool +``synapse.tools.backup`` may be used to create a backup copy of this working directory which may then be restored. + +It is important that you use ``synapse.tools.backup`` rather than simply copying the directory. + +It is also worth noting that the newly created backup is a defragmented / optimized copy of all database data +structures. As such, we recommend occasionally scheduling a maintenance window to create a "cold backup" with the +service offline and deploy the backup copy when bringing the service back online. Regularly performing this +"restore from cold backup" procedure can dramatically improve performance and resource utilization. + +Running A Backup +**************** + +Continuing our previous example, running a backup is as simple as:: + python -m synapse.tools.backup /data/cortex00 /backups/cortex00_`date +%Y%m%d` + +Assuming that your backup was run on ``May 19, 2020``, this would create a backup in the directory ``/backups/cortex00_20200519``. + +The backup command can be run on a live service. Depending on your configuration, creating a live backup +of your service can temporarily degrade performance of the running service. As such, it may be best to schedule +backups during low utilization time windows. + +Restoring From Backup +********************* + +In the event that restoring from backup is necessary, simply move the service working directory and +copy a previous backup directory to the service working directory location. From our previous example, +this would involve running the following shell commands:: + mv /data/cortex00 /data/cortex00_old + cp -R /backups/cortex00_20200519 /data/cortex00 + +TLS/SSL Deployments +------------------- + +For production deployments, it is recommended that all services use the built-in ``telepath`` SSL/TLS +protocol. You may deploy a service using TLS encryption by specifying a ``--telepath`` listen URL option, such +as ``ssl://cortex.vertex.link/``. + +Under some circumstances, such as inbound DNAT networks or multi-homed hosts, it may be necessary to specify a +socket bind address that is different than the hostname. If your environment requires you to bind an address that +is different than the hostname's DNS entry, an explicit hostname query parameter may be +specified:``ssl://0.0.0.0/?hostname=cortex.vertex.link``. + +The client will also need to specify an SSL/TLS ``telepath`` URL such as: ``ssl://visi:passwd@cortex.vertex.link``. + +Once the ``ssl`` protocol is specified, the ``hostname``, either from a query parameter or from the URL's +network location, is used to lookup a matching ``crt`` and ``key`` file pair from your server certificate directory +located at ``~/.syn/certs/hosts``. See the following sections for how to setup server certificates. + +Self-Signed Certificates using certtool +*************************************** + +For self-signed certificates, we need to generate a CA certificate and key as well as a server certificate and key. + +The synapse ``certtool`` can be used to easily generate a CA and server certificates. For example, if we wanted +to generate a CA certificate for "vertex":: + + python -m synapse.tools.easycert --ca vertex + key saved: /home/visi/.syn/certs/cas/vertex.key + cert saved: /home/visi/.syn/certs/cas/vertex.crt + +We can then generate a server certificate and keyfile pair and sign it with our new CA:: + + python -m synapse.tools.easycert --server cortex.vertex.link --signas vertex + key saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.key + cert saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.crt + +To verify the server certificate, clients will need to have the ``~/.syn/certs/cas/vertex.crt`` file in their +certificate directory. + +NOTE: do not distribute the ``~/.syn/certs/cas/vertex.key`` file as that would allow regular users the ability +to sign arbitrary certificates). + +CA-Signed Certificates +********************** + +Any external CA may be used to sign ``telepath`` server certificates. The ``certtool`` can be used to easily +generate a certificate signing request (CSR) to be signed by an external CA or you can simply copy or link +pre-existing PEM encoded certificate files to the expected filesystem locations. + +To generate a CSR using ``certtool``:: + python -m synapse.tools.easycert --csr --server cortex.vertex.link + key saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.key + csr saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.csr + +You may then submit your CSR file (in this case ``~/.syn/certs/hosts/cortex.vertex.link.csr``) to your CA of choice for signing. +Once your CA returns a signed certificate in PEM format, place it in the expected location (``~/.syn/certs/hosts/cortex.vertex.link.crt`` in this example) +and it will be loaded when you start your service. + +Client-Side Certificates for Authentication +******************************************* + +TODO diff --git a/docs/synapse/devops.rst b/docs/synapse/devops.rst index a46a9a161e..1bd1367207 100644 --- a/docs/synapse/devops.rst +++ b/docs/synapse/devops.rst @@ -6,219 +6,19 @@ Synapse DevOps Guide #################### +The DevOps guide contains information useful for running **Synapse** ecosystem components. This will continue to be +updated and expanded over time. .. toctree:: :titlesonly: + devguides/devops_general + devguides/devops_cortex + devguides/devops_axon + devguides/devops_cryotank + autodocs/conf_axon autodocs/conf_cortex autodocs/conf_cryocell - -General -======= - -Backups -------- - -It is strongly recommended that users schedule regular backups of all services deployed within their **Synapse** -ecosystem. Each service can be backed up using the **Synapse** backup tool: ``synapse.tools.backup``. - -The **Synapse** service architecture is designed to contain everything a service needs within the directory you -specify during service startup. Take, for example, a **Cortex** started with:: - python -m synapse.servers.cortex /data/cortex00 - -The **Cortex** will be completely contained within the service working directory ``/data/cortex00``. The synapse tool -``synapse.tools.backup`` may be used to create a backup copy of this working directory which may then be restored. - -It is important that you use ``synapse.tools.backup`` rather than simply copying the directory. - -It is also worth noting that the newly created backup is a defragmented / optimized copy of all database data -structures. As such, we recommend occasionally scheduling a maintenance window to create a "cold backup" with the -service offline and deploy the backup copy when bringing the service back online. Regularly performing this -"restore from cold backup" procedure can dramatically improve performance and resource utilization. - -Running A Backup -**************** - -Continuing our previous example, running a backup is as simple as:: - python -m synapse.tools.backup /data/cortex00 /backups/cortex00_`date +%Y%m%d` - -Assuming that your backup was run on ``May 19, 2020``, this would create a backup in the directory ``/backups/cortex00_20200519``. - -The backup command can be run on a live service. Depending on your configuration, creating a live backup -of your service can temporarily degrade performance of the running service. As such, it may be best to schedule -backups during low utilization time windows. - -Restoring From Backup -********************* - -In the event that restoring from backup is necessary, simply move the service working directory and -copy a previous backup directory to the service working directory location. From our previous example, -this would involve running the following shell commands:: - mv /data/cortex00 /data/cortex00_old - cp -R /backups/cortex00_20200519 /data/cortex00 - -TLS/SSL Deployments -------------------- - -For production deployments, it is recommended that all services use the built-in ``telepath`` SSL/TLS -protocol. You may deploy a service using TLS encryption by specifying a ``--telepath`` listen URL option, such -as ``ssl://cortex.vertex.link/``. - -Under some circumstances, such as inbound DNAT networks or multi-homed hosts, it may be necessary to specify a -socket bind address that is different than the hostname. If your environment requires you to bind an address that -is different than the hostname's DNS entry, an explicit hostname query parameter may be -specified:``ssl://0.0.0.0/?hostname=cortex.vertex.link``. - -The client will also need to specify an SSL/TLS ``telepath`` URL such as: ``ssl://visi:passwd@cortex.vertex.link``. - -Once the ``ssl`` protocol is specified, the ``hostname``, either from a query parameter or from the URL's -network location, is used to lookup a matching ``crt`` and ``key`` file pair from your server certificate directory -located at ``~/.syn/certs/hosts``. See the following sections for how to setup server certificates. - -Self-Signed Certificates using certtool -*************************************** - -For self-signed certificates, we need to generate a CA certificate and key as well as a server certificate and key. - -The synapse ``certtool`` can be used to easily generate a CA and server certificates. For example, if we wanted -to generate a CA certificate for "vertex":: - - python -m synapse.tools.easycert --ca vertex - key saved: /home/visi/.syn/certs/cas/vertex.key - cert saved: /home/visi/.syn/certs/cas/vertex.crt - -We can then generate a server certificate and keyfile pair and sign it with our new CA:: - - python -m synapse.tools.easycert --server cortex.vertex.link --signas vertex - key saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.key - cert saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.crt - -To verify the server certificate, clients will need to have the ``~/.syn/certs/cas/vertex.crt`` file in their -certificate directory. - -NOTE: do not distribute the ``~/.syn/certs/cas/vertex.key`` file as that would allow regular users the ability -to sign arbitrary certificates). - -CA-Signed Certificates -********************** - -Any external CA may be used to sign ``telepath`` server certificates. The ``certtool`` can be used to easily -generate a certificate signing request (CSR) to be signed by an external CA or you can simply copy or link -pre-existing PEM encoded certificate files to the expected filesystem locations. - -To generate a CSR using ``certtool``:: - python -m synapse.tools.easycert --csr --server cortex.vertex.link - key saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.key - csr saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.csr - -You may then submit your CSR file (in this case ``~/.syn/certs/hosts/cortex.vertex.link.csr``) to your CA of choice for signing. -Once your CA returns a signed certificate in PEM format, place it in the expected location (``~/.syn/certs/hosts/cortex.vertex.link.crt`` in this example) -and it will be loaded when you start your service. - -Client-Side Certificates for Authentication -******************************************* - -TODO - -Cortex -====== - -Docker Deployment ------------------ - -Configuring A Mirror --------------------- - -.. _200_migration: - -0.1.x to 2.x.x Migration ------------------------- - -Two tools have been created to execute migration of an ``0.1.x`` Cortex to ``2.x.x``: - -* ``migrate_200`` migrates all data from the source to a new destination ``2.x.x`` Cortex. -* ``sync_200`` allows for a backup to be migrated and then synchronized with a running Cortex to facilitate minimal downtime. - -Migration Quickstart -******************** - -.. note:: - - The duration of migration is proportional to the amount of data stored, and is highly dependent on - the available system resources (especially disk I/O). For larger Cortexes it is recommended to - run migration on hardware without other highly active processes. - -#. Update the source to the latest Synapse ``0.1.x`` release. -#. Create a backup of the ``0.1.x`` Cortex. -#. In a new location install Synapse ``2.x.x`` and copy any custom modules / configurations present in the ``0.1.x`` environment. -#. Start migration using the backup as the source:: - - python -m synapse.tools.migrate_200 --src --dest - -#. Inspect the migration output for any errors that may require action (see :ref:`migration-errors` for details). -#. Startup the ``2.x.x`` Cortex. - -Migration Options -***************** - -* ``--from-last`` restarts node migration from the last checkpoint (automatically saved at periodic intervals). -* ``--safety-off`` disables form value normalization checks as a pre-condition to migrate nodes (may allow migration to run faster). -* ``--src-dedicated`` opens the source layer slabs with locked memory (must have sufficient memory available). - -Additional options that, if specified, will *not* run a migration process: - -* ``--form-counts`` is a helper utility that scans all nodes and produces counts by form for source and destination. -* ``--dump-errors`` saves migration errors to a file in the migration directory in msgpack format. - -.. _migration-errors: - -Migration Errors -**************** - -During node migration the following errors may occur, all of which indicate that the node was not migrated: - -``Unable to determine stortype`` or ``Buid/norming exception: NoSuchForm`` - -* A custom form/property may not have been properly loaded into the ``2.x.x`` environment. -* The node may not have been properly updated in a prior ``0.1.x`` datamodel migration and therefore no longer exists. - -``Normed form val does not match inbound`` or ``Calculated buid does not match inbound`` - -* Is likely due to a node that was not properly re-normalized after a prior Synapse update. - -Post-migration Synchronization -****************************** - -After migration, the ``sync_200`` service can be used to push post-backup changes to the migrated ``2.x.x`` Cortex, -and keep it updated until cut-over. ``sync_200`` uses splices to translate the changes, and therefore they must -be enabled on the source Cortex. In order to control and monitor synchronization, ``sync_200`` can be added as a Storm service. - -When synchronization is started the service will enable "migration mode" on the destination ``2.x.x`` Cortex, which -prevents cron jobs and triggers from running. Migration mode will then be disabled when the synchronization is -stopped or when the Cortex is restarted. - -#. Complete migration, including starting up the ``2.x.x`` Cortex. -#. Locate the saved splice offset file from migration at ``/migration/lyroffs.yaml``. -#. Start the ``sync_200`` service (shown with the optional ``--auth-passwd`` to bootstrap the root user):: - - python -m synapse.tools.sync_200 \ - --auth-passwd secret --offsfile \ - --src <01x_telepath_url> --dest <20x_telepath_url> - -#. Add the Storm service to the Cortex and use the available commands to start synchronization. -#. When ready to cut-over, and the read status is up-to-date, stop the synchronization using the ``stopsync`` command. - -Cleanup -******* - -After migration is fully complete, delete the now-unused directory "migration" inside the cortex directory. - -Axon -==== - -Cryotank -======== - .. _index: ../index.html diff --git a/synapse/tools/autodoc.py b/synapse/tools/autodoc.py index aee345683f..ce99d8da0b 100644 --- a/synapse/tools/autodoc.py +++ b/synapse/tools/autodoc.py @@ -352,16 +352,13 @@ async def docModel(outp, async def docConfdefs(ctor, reflink=':ref:`devops_cell_config`'): cls = s_dyndeps.tryDynLocal(ctor) - print(cls) - if not hasattr(cls, 'confdefs'): raise Exception('ctor must have a confdefs attr') rst = RstHelp() - print(cls.confdefs) + clsname = cls.__name__ conf = cls.initCellConf() # type: s_config.Config - print(conf) rst.addHead(f'{clsname} Configuration Options', lvl=0) rst.addLines(f'The following are boot-time configuration options for a {clsname}') @@ -381,7 +378,6 @@ async def docConfdefs(ctor, reflink=':ref:`devops_cell_config`'): for name, conf in sorted(schema.items(), key=lambda x: x[0]): nodesc = f'No description available for ``{name}``.' - print(name, conf) hname = name if ':' in name: hname = name.replace(':', raw_back_slash_colon) @@ -389,9 +385,17 @@ async def docConfdefs(ctor, reflink=':ref:`devops_cell_config`'): rst.addHead(hname, lvl=1) desc = conf.get('description', nodesc) + if not desc.endswith('.'): # pragma: no cover + logger.warning(f'Description for [{name}] is missing a period.') + lines = [] lines.append(desc) + extended_description = conf.get('extended_description') + if extended_description: + lines.append('\n') + lines.append(extended_description) + # Type/additional information lines.append('\n') @@ -449,14 +453,10 @@ async def main(argv, outp=None): if opts.doc_cell: confdocs, cname = await docConfdefs(opts.doc_cell) - print(confdocs, cname) - if opts.savedir: with open(s_common.genpath(opts.savedir, f'conf_{cname.lower()}.rst'), 'wb') as fd: fd.write(confdocs.getRstText().encode()) - print(confdocs.getRstText()) - return 0 def makeargparser(): From a8c507649270a41c0c0dce61ce5d26ea0cebd6dc Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 15 Jun 2020 18:23:49 +0000 Subject: [PATCH 05/13] Fix parameter --- synapse/tools/autodoc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/tools/autodoc.py b/synapse/tools/autodoc.py index ce99d8da0b..52611fca94 100644 --- a/synapse/tools/autodoc.py +++ b/synapse/tools/autodoc.py @@ -364,7 +364,7 @@ async def docConfdefs(ctor, reflink=':ref:`devops_cell_config`'): rst.addLines(f'The following are boot-time configuration options for a {clsname}') # FIXME Make this a parameter so we can link from non-synapse stacks here. - rst.addLines(f'See :ref:`devops_cell_config` for details on how to set these options..') + rst.addLines(f'See {reflink} for details on how to set these options.') # access raw config data @@ -451,7 +451,8 @@ async def main(argv, outp=None): fd.write(rstforms.getRstText().encode()) if opts.doc_cell: - confdocs, cname = await docConfdefs(opts.doc_cell) + confdocs, cname = await docConfdefs(opts.doc_cell, + reflink=opts.doc_cell_reflink) if opts.savedir: with open(s_common.genpath(opts.savedir, f'conf_{cname.lower()}.rst'), 'wb') as fd: @@ -472,6 +473,8 @@ def makeargparser(): help='Generate RST docs for the DataModel within a cortex') doc_type.add_argument('--doc-cell', default=None, help='Generate RST docs for the Confdefs for a given Cell ctor') + pars.add_argument('--doc-cell-reflink', default=':ref:`devops_cell_config`', + help='Reference link for how to set the cell configuration options.') return pars From 73e1d5323464cf410a9c64e686f5306243ce3b19 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 15 Jun 2020 18:24:07 +0000 Subject: [PATCH 06/13] Remove comment --- synapse/tools/autodoc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/tools/autodoc.py b/synapse/tools/autodoc.py index 52611fca94..4d92350fc1 100644 --- a/synapse/tools/autodoc.py +++ b/synapse/tools/autodoc.py @@ -363,7 +363,6 @@ async def docConfdefs(ctor, reflink=':ref:`devops_cell_config`'): rst.addHead(f'{clsname} Configuration Options', lvl=0) rst.addLines(f'The following are boot-time configuration options for a {clsname}') - # FIXME Make this a parameter so we can link from non-synapse stacks here. rst.addLines(f'See {reflink} for details on how to set these options.') # access raw config data From 7f0c54a6dbfb5ac101b1b5fc000a5dc401bb4352 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 15 Jun 2020 18:34:17 +0000 Subject: [PATCH 07/13] Add sync migrator to the confdefs list --- docs/conf.py | 6 +++++- docs/synapse/devops.rst | 1 + synapse/cortex.py | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 10a7df1736..dc6741ff32 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -216,7 +216,11 @@ def run_confdocs(_): synpd = os.path.split(synbd)[0] # split off the synapse module directory baseargs = ['python', '-m', 'synapse.tools.autodoc', '--savedir', './docs/synapse/autodocs', '--doc-cell'] - ctors = ('synapse.axon.Axon', 'synapse.cortex.Cortex', 'synapse.cryotank.CryoCell') + ctors = ('synapse.axon.Axon', + 'synapse.cortex.Cortex', + 'synapse.cryotank.CryoCell', + 'synapse.tools.sync_200.SyncMigrator', + ) for ctor in ctors: args = baseargs.copy() args.append(ctor) diff --git a/docs/synapse/devops.rst b/docs/synapse/devops.rst index 1bd1367207..754f7fa02d 100644 --- a/docs/synapse/devops.rst +++ b/docs/synapse/devops.rst @@ -20,5 +20,6 @@ updated and expanded over time. autodocs/conf_axon autodocs/conf_cortex autodocs/conf_cryocell + autodocs/conf_syncmigrator .. _index: ../index.html diff --git a/synapse/cortex.py b/synapse/cortex.py index 6d11c0d970..0a38e0979b 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -702,7 +702,7 @@ class Cortex(s_cell.Cell): # type: ignore 'type': 'string' }, 'mirror': { - 'description': 'Run a mirror of the cortex at the given telepath URL. (we must be a backup!)', + 'description': 'Run a mirror of the cortex at the given telepath URL. We must be a backup!', 'type': 'string' }, 'cron:enable': { @@ -732,7 +732,7 @@ class Cortex(s_cell.Cell): # type: ignore }, 'provenance:en': { 'default': False, - 'description': 'Enable provenance tracking for all writes', + 'description': 'Enable provenance tracking for all writes.', 'type': 'boolean' }, 'modules': { From 48837e1664f060182cbc11c3577696edce24b534 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 15 Jun 2020 18:36:09 +0000 Subject: [PATCH 08/13] Fix comments --- synapse/tools/autodoc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/tools/autodoc.py b/synapse/tools/autodoc.py index 4d92350fc1..3a5417bc60 100644 --- a/synapse/tools/autodoc.py +++ b/synapse/tools/autodoc.py @@ -367,10 +367,9 @@ async def docConfdefs(ctor, reflink=':ref:`devops_cell_config`'): # access raw config data - # Get raw envars + # Get envar and argparse mapping name2envar = conf.getEnvarMapping() name2cmdline = conf.getCmdlineMapping() - # Get argparse mappping? schema = conf.json_schema.get('properties', {}) From 026cd3aaa0e98d27573a2227d7a9b7ef86b4681f Mon Sep 17 00:00:00 2001 From: mikemoritz <57907149+mikemoritz@users.noreply.github.com> Date: Tue, 16 Jun 2020 14:35:05 -0400 Subject: [PATCH 09/13] Update storm service docs (#1771) * update storm service docs * split out cell config docs * move cell config to its own file * Add unit tests for new autodoc functionality. Co-authored-by: epiphyte --- docs/conf.py | 2 +- docs/synapse/devguides/devops_cell.rst | 89 ++++++++++++++++++++++++ docs/synapse/devguides/stormservices.rst | 86 ++++++++++++++++++++--- docs/synapse/devops.rst | 1 + synapse/tests/test_tools_autodoc.py | 36 ++++++++++ synapse/tools/autodoc.py | 12 ++-- 6 files changed, 210 insertions(+), 16 deletions(-) create mode 100644 docs/synapse/devguides/devops_cell.rst diff --git a/docs/conf.py b/docs/conf.py index dc6741ff32..e3ab825fa0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -215,7 +215,7 @@ def run_confdocs(_): synbd = os.path.split(abssynf)[0] # Split off __init__ synpd = os.path.split(synbd)[0] # split off the synapse module directory baseargs = ['python', '-m', 'synapse.tools.autodoc', '--savedir', - './docs/synapse/autodocs', '--doc-cell'] + './docs/synapse/autodocs', '--doc-conf'] ctors = ('synapse.axon.Axon', 'synapse.cortex.Cortex', 'synapse.cryotank.CryoCell', diff --git a/docs/synapse/devguides/devops_cell.rst b/docs/synapse/devguides/devops_cell.rst new file mode 100644 index 0000000000..f813cbbcec --- /dev/null +++ b/docs/synapse/devguides/devops_cell.rst @@ -0,0 +1,89 @@ +Cell Operations +=============== + +As detailed in :ref:`dev_architecture`, the ``Cell`` implements a number of core management functionalities, +and is therefore used as the base class for Synapse applications. It is also recommended to implement ``Cell`` for custom services. + +.. _devops-cell-config: + +Configuring a Cell Service +-------------------------- + +A Cell has a set of base configuration data that will also be inherited by any implementations: + +- ``dirn``: The storage directory for the Cell which is a required argument for service startup. +- ``telepath``: Optional override for the Telepath URL to listen on. +- ``https``: Optional override for the port to bind for the HTTPS/REST API. +- ``name``: Optional additional name to share the service as. + +The Cell class also specifies configuration variables in ``confdefs``: + +- ``auth:passwd``: Optional bootstrapping for the root user password. +- ``nexslog:en``: Optional enablement of Nexus logging (most custom Cell implementations will not set this). + +Cell implementations can extend the configuration variables available by specifying them in +``confdefs`` in the Cell subclass. The variable names can be namespaced using colons, for example ``mysvc:apikey``. + +Depending on deployment requirements, a combination of methods can be used for loading the configurations into the Cell. + +Config File +*********** + +A Cell has one optional configuration file, ``cell.yaml``, that may be located in the root Cell directory. +The format of this file is YAML, and variable names are specified without alteration, for example:: + + --- + auth:passwd: secret + mysvc:apikey: 720a50f9-cfa1-43a9-9eca-dda379ecd8c5 + --- + +Environment Variables +********************* + +Environment variable names are automatically generated for a Cell service using the following naming convention: +``SYN__``. Variable names with colons are replaced with underscores, +and the raw environmental variable value is deserialized as yaml, prior to performing type validation. + +Command Line +************ + +Variables which can only be easily passed as command line arguments are available on the command line. +Variable names with colons are replaced with a single dash. + +.. note:: + + When used to launch a Cell as the entry point of a program, the configuration precedence order is: + + #. Command line arguments + #. Environment variables + #. cell.yaml values + + These may all be mixed and matched for a given deployment. + If a backup of a service is made and the deployment uses configuration data from command line arguments and environmental variables, + those will need to be considered when moving/restoring the backup. + +Starting a Cell Service +----------------------- + +The examples provided below are intended for Cell implementations outside of the Synapse level components, +which have their own servers in the ``synapse.servers`` module. + +As Main Module +************** + +Cell implementations may define the following as the main application entrypoint (where MySvc is the Cell subclass):: + + if __name__ == '__main__': + asyncio.run(MySvc.execmain(sys.argv[1:])) + +The service can then be started with:: + + python -m path.to.main /path/to/dirn + +As Cell Server +************** + +The generic Cell server can also be used for starting the service by specifying the constructor as an argument:: + + python -m synapse.servers.cell path.to.MySvc /path/to/dirn + diff --git a/docs/synapse/devguides/stormservices.rst b/docs/synapse/devguides/stormservices.rst index bd9d677666..13906a326d 100644 --- a/docs/synapse/devguides/stormservices.rst +++ b/docs/synapse/devguides/stormservices.rst @@ -3,21 +3,69 @@ Storm Service Development ######################### -NOTE: These docs are an initial place holder to hold some notes. - Anatomy of a Storm Service ========================== -Storm Service Modules -===================== +A Storm Service (see :ref:`gloss-service`) is a standalone application that extends the capabilities of the Cortex. +One common use case for creating a service is to add a Storm command that will query third-party data, +translate the results into the Synapse datamodel, and then ingest them into the hypergraph. + +In order to leverage core functionalities it is recommended that Storm services are created as Cell implementations, +and the documentation that follows will assume this. For additional information see :ref:`dev_architecture`. + +A Storm service generally implements the following components: + +- A :ref:`gloss-package` that contains the new :ref:`stormservice-cmd` and optional new :ref:`stormservice-mod`. + +- A subclass of ``synapse.lib.CellApi`` which uses the ``synapse.lib.StormSvc`` mixin and contains the following information: + + - The service name, version, packages, and events as defined in ``synapse.lib.StormSvc``. + - Custom methods which will be accessible as Telepath API endpoints, and therefore available for use within defined Storm commands. + +- A subclass of ``synapse.lib.Cell`` which includes additional configuration defintions and methods required to implement the service. + +Connecting a service +-------------------- + +For instructions on configuring and starting a Cell service see :ref:`devops-cell-config`. + +Before connecting a service to a Cortex it is best practice to add a new user to the service Cell (see :ref:`initial-roles`). + +A Storm command can then be run on the Cortex to add the new service:: + + service.add mysvc tcp://:@:27495 + +Permissions to access the service can be granted by adding the ``service.get.`` rule to the appropriate users / roles. + +The new Storm commands will now be available for use, and are included in Storm ``help``. + +.. _stormservice-cmd: Storm Service Commands ====================== +Implementation +-------------- + +Multiple Storm commands can be added to a Storm service package, with each defining the following attributes: + + - ``name``: Name of the Storm command to surface in the Cortex. + - ``descr``: Description of the command which will be available in ``help`` displays. + - ``cmdargs``: An optional list of arguments for the command. + - ``cmdconf``: An optional dictionary of additional configuration variables to provide to the command Storm execution. + - ``forms``: List of input and output forms for the command. + - ``storm``: The Storm code, as a string, that will be executed when the command is called. + +Typically, the Storm code will start by getting a reference to the service via ``$svc = $lib.service.get($cmdconf.svciden)`` +and reading in any defined ``cmdargs`` that are available in ``$cmdopts``. The methods defined in the service's Cell API +can then be called by, for example, ``$retn = $svc.mysvcmethod($cmdopts.query)``. + Input/Output Conventions ------------------------ -Most commands that enrich or add additional context to nodes should simply yield the nodes they were given as inputs. If they don’t know how to enrich or add additional context to a given form, nodes of that form should be yielded rather than producing an error. This allows a series of enrichment commands to be pipelined regardless of the different inputs that a given command knows how to operate on. +Most commands that enrich or add additional context to nodes should simply yield the nodes they were given as inputs. +If they don’t know how to enrich or add additional context to a given form, nodes of that form should be yielded rather than producing an error. +This allows a series of enrichment commands to be pipelined regardless of the different inputs that a given command knows how to operate on. Argument Conventions -------------------- @@ -25,16 +73,36 @@ Argument Conventions ``--verbose`` ~~~~~~~~~~~~~ -In general, storm commands should operate silently over their input nodes and should especially avoid printing anything "per node". However, when an error occurs, the command may use ``$lib.warn()`` to print a warning message per-node. Commands should implement a ``--verbose`` command line option to enable printing "per node" informational output. +In general, Storm commands should operate silently over their input nodes and should especially avoid printing anything "per node". +However, when an error occurs, the command may use ``$lib.warn()`` to print a warning message per-node. +Commands should implement a ``--verbose`` command line option to enable printing "per node" informational output. ``--debug`` ~~~~~~~~~~~ +For commands where additional messaging would assist in debugging a ``--debug`` command line option should be implemented. +For example, a Storm command that is querying a third-party data source could use ``$lib.print()`` to print the raw query string +and raw response when the ``--debug`` option is specified. + ``--yield`` ~~~~~~~~~~~ -For commands that create additional nodes, it may be beneficial to add a --yield option to allow a query to operate on the newly created nodes. Some guidelines for ``--yield`` options: +For commands that create additional nodes, it may be beneficial to add a ``--yield`` option to allow a query to operate on the newly created nodes. +Some guidelines for ``--yield`` options: -- The command should *not* yield the input node(s) when a --yield is specified -- The --yield option should *not* be implemented when pivoting from the input node to reach the newly created node is a “refs out” or 1-to-1 direct pivot. For example, there is no need to have a --yield option on the ``maxmind`` command even though it may create an ``inet:asn`` node for an input ``inet:ipv4`` node due to the 1-to-1 pivot ``-> inet:asn`` being possible. +- The command should *not* yield the input node(s) when a ``--yield`` is specified +- The ``--yield`` option should *not* be implemented when pivoting from the input node to reach the newly created node is a “refs out” or 1-to-1 direct pivot. For example, there is no need to have a ``--yield`` option on the ``maxmind`` command even though it may create an ``inet:asn`` node for an input ``inet:ipv4`` node due to the 1-to-1 pivot ``-> inet:asn`` being possible. - The ``--yield`` option should ideally determine a “primary” node form to yield even when the command may create many forms in order to tag them or update .seen times. + +.. _stormservice-mod: + +Storm Service Modules +===================== + +Modules can be added to a Storm service package to surface reusable Storm functions. +Each module defines a ``name``, which is used for importing elsewhere via ``$lib.import()``, +and a ``storm`` string. The Storm code in this case contains callable functions with the format:: + + function myfunc(var1, var2) { + // function Storm code + } diff --git a/docs/synapse/devops.rst b/docs/synapse/devops.rst index 754f7fa02d..9fcf030561 100644 --- a/docs/synapse/devops.rst +++ b/docs/synapse/devops.rst @@ -13,6 +13,7 @@ updated and expanded over time. :titlesonly: devguides/devops_general + devguides/devops_cell devguides/devops_cortex devguides/devops_axon devguides/devops_cryotank diff --git a/synapse/tests/test_tools_autodoc.py b/synapse/tests/test_tools_autodoc.py index ab78e67674..138c60aded 100644 --- a/synapse/tests/test_tools_autodoc.py +++ b/synapse/tests/test_tools_autodoc.py @@ -34,3 +34,39 @@ async def test_tools_autodoc_docmodel(self): self.isin('Universal props are system level properties which may be present on every node.', s) self.isin('.created', s) self.notin('..created\n', s) + + async def test_tools_autodoc_confdefs(self): + + with self.getTestDir() as path: + + argv = ['--savedir', path, '--doc-conf', + 'synapse.tests.test_lib_stormsvc.StormvarServiceCell'] + + outp = self.getTestOutp() + self.eq(await s_autodoc.main(argv, outp=outp), 0) + + with s_common.genfile(path, 'conf_stormvarservicecell.rst') as fd: + buf = fd.read() + s = buf.decode() + + self.isin('StormvarServiceCell Configuration Options', s) + self.isin('See :ref:`devops-cell-config` for', s) + self.isin('auth\:passwd', s) + self.isin('Environment Variable\n ``SYN_STORMVARSERVICECELL_AUTH_PASSWD``', s) + self.isin('``--auth-passwd``', s) + + argv.append('--doc-conf-reflink') + argv.append('`Configuring a Cell Service `_') + + # truncate the current file + with s_common.genfile(path, 'conf_stormvarservicecell.rst') as fd: + fd.truncate() + + outp = self.getTestOutp() + self.eq(await s_autodoc.main(argv, outp=outp), 0) + with s_common.genfile(path, 'conf_stormvarservicecell.rst') as fd: + buf = fd.read() + s = buf.decode() + + self.isin('StormvarServiceCell Configuration Options', s) + self.isin('See `Configuring a Cell Service Date: Tue, 16 Jun 2020 20:23:26 +0000 Subject: [PATCH 10/13] Docs stormsvc cmds (#1773) * Add autodocs support for generating rst from a stormservice definition Co-authored-by: redox --- synapse/tests/test_lib_stormsvc.py | 7 +- synapse/tests/test_tools_autodoc.py | 24 ++++++ synapse/tools/autodoc.py | 123 ++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 1 deletion(-) diff --git a/synapse/tests/test_lib_stormsvc.py b/synapse/tests/test_lib_stormsvc.py index 959c28b5fd..9ba3a3c1bd 100644 --- a/synapse/tests/test_lib_stormsvc.py +++ b/synapse/tests/test_lib_stormsvc.py @@ -245,7 +245,7 @@ class LifterService(s_stormsvc.StormSvc): 'commands': ( { 'name': 'lifter', - 'desc': 'Lift inet:ipv4=1.2.3.4', + 'descr': 'Lift inet:ipv4=1.2.3.4', 'storm': 'inet:ipv4=1.2.3.4', }, ), @@ -269,10 +269,15 @@ class StormvarService(s_cell.CellApi, s_stormsvc.StormSvc): 'commands': ( { 'name': 'magic', + 'descr': 'Test stormvar support.', 'cmdargs': ( ('name', {}), ('--debug', {'default': False, 'action': 'store_true'}) ), + 'forms': { + 'input': ('test:str', 'test:int'), + 'output': ('test:comp', 'inet:ipv4'), + }, 'storm': ''' $fooz = $cmdopts.name if $cmdopts.debug { diff --git a/synapse/tests/test_tools_autodoc.py b/synapse/tests/test_tools_autodoc.py index 138c60aded..01bd0075eb 100644 --- a/synapse/tests/test_tools_autodoc.py +++ b/synapse/tests/test_tools_autodoc.py @@ -70,3 +70,27 @@ async def test_tools_autodoc_confdefs(self): self.isin('StormvarServiceCell Configuration Options', s) self.isin('See `Configuring a Cell Service Date: Tue, 16 Jun 2020 21:14:21 +0000 Subject: [PATCH 11/13] Fix yaml and typos --- docs/synapse/devguides/devops_cell.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/synapse/devguides/devops_cell.rst b/docs/synapse/devguides/devops_cell.rst index f813cbbcec..21ee1eb833 100644 --- a/docs/synapse/devguides/devops_cell.rst +++ b/docs/synapse/devguides/devops_cell.rst @@ -2,7 +2,8 @@ Cell Operations =============== As detailed in :ref:`dev_architecture`, the ``Cell`` implements a number of core management functionalities, -and is therefore used as the base class for Synapse applications. It is also recommended to implement ``Cell`` for custom services. +and is therefore used as the base class for Synapse applications. It is also recommended to implement ``Cell`` for +custom services. .. _devops-cell-config: @@ -35,14 +36,14 @@ The format of this file is YAML, and variable names are specified without altera --- auth:passwd: secret mysvc:apikey: 720a50f9-cfa1-43a9-9eca-dda379ecd8c5 - --- + ... Environment Variables ********************* Environment variable names are automatically generated for a Cell service using the following naming convention: ``SYN__``. Variable names with colons are replaced with underscores, -and the raw environmental variable value is deserialized as yaml, prior to performing type validation. +and the raw environment variable value is deserialized as yaml, prior to performing type validation. Command Line ************ @@ -59,8 +60,8 @@ Variable names with colons are replaced with a single dash. #. cell.yaml values These may all be mixed and matched for a given deployment. - If a backup of a service is made and the deployment uses configuration data from command line arguments and environmental variables, - those will need to be considered when moving/restoring the backup. + If a backup of a service is made and the deployment uses configuration data from command line arguments and + environment variables, those will need to be considered when moving/restoring the backup. Starting a Cell Service ----------------------- From 0bd42ce242643531b10d7e758642b2ccd8965ab0 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Tue, 16 Jun 2020 21:39:32 +0000 Subject: [PATCH 12/13] Fix autodoc test --- synapse/tests/test_tools_autodoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/tests/test_tools_autodoc.py b/synapse/tests/test_tools_autodoc.py index 01bd0075eb..bf0089ab14 100644 --- a/synapse/tests/test_tools_autodoc.py +++ b/synapse/tests/test_tools_autodoc.py @@ -85,7 +85,7 @@ async def test_tools_autodoc_stormsvc(self): buf = fd.read() s = buf.decode() - self.isin('Storm Service - StormvarServiceCell', s) + self.isin('StormvarServiceCell Storm Service', s) self.isin('This documentation is generated for version 0.0.1 of the service.', s) self.isin('Storm Package\: stormvar', s) self.isin('magic\n-----', s) From ed05d728ef62effe8089a323a5b83699d13301b1 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Tue, 16 Jun 2020 21:40:14 +0000 Subject: [PATCH 13/13] Surface -> expose --- docs/synapse/devguides/stormservices.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/synapse/devguides/stormservices.rst b/docs/synapse/devguides/stormservices.rst index 13906a326d..180e89f4a7 100644 --- a/docs/synapse/devguides/stormservices.rst +++ b/docs/synapse/devguides/stormservices.rst @@ -49,7 +49,7 @@ Implementation Multiple Storm commands can be added to a Storm service package, with each defining the following attributes: - - ``name``: Name of the Storm command to surface in the Cortex. + - ``name``: Name of the Storm command to expose in the Cortex. - ``descr``: Description of the command which will be available in ``help`` displays. - ``cmdargs``: An optional list of arguments for the command. - ``cmdconf``: An optional dictionary of additional configuration variables to provide to the command Storm execution. @@ -99,7 +99,7 @@ Some guidelines for ``--yield`` options: Storm Service Modules ===================== -Modules can be added to a Storm service package to surface reusable Storm functions. +Modules can be added to a Storm service package to expose reusable Storm functions. Each module defines a ``name``, which is used for importing elsewhere via ``$lib.import()``, and a ``storm`` string. The Storm code in this case contains callable functions with the format::