diff --git a/docs/conf.py b/docs/conf.py index 2ff75d8518..e3ab825fa0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -208,6 +208,24 @@ def run_modeldoc(_): '--savedir', './docs/synapse/autodocs'] subprocess.run(args, cwd=synpd) +def run_confdocs(_): + import synapse + import subprocess + abssynf = os.path.abspath(synapse.__file__) + synbd = os.path.split(abssynf)[0] # Split off __init__ + synpd = os.path.split(synbd)[0] # split off the synapse module directory + baseargs = ['python', '-m', 'synapse.tools.autodoc', '--savedir', + './docs/synapse/autodocs', '--doc-conf'] + ctors = ('synapse.axon.Axon', + 'synapse.cortex.Cortex', + 'synapse.cryotank.CryoCell', + 'synapse.tools.sync_200.SyncMigrator', + ) + for ctor in ctors: + args = baseargs.copy() + args.append(ctor) + subprocess.run(args, cwd=synpd) + def convert_ipynb(_): import synapse.common as s_common import nbconvert.nbconvertapp as nba @@ -228,4 +246,5 @@ def convert_ipynb(_): def setup(app): app.connect('builder-inited', run_apidoc) app.connect('builder-inited', run_modeldoc) + app.connect('builder-inited', run_confdocs) app.connect('builder-inited', convert_ipynb) diff --git a/docs/synapse/devguides/backups.rst b/docs/synapse/devguides/backups.rst new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/synapse/devguides/devops_axon.rst b/docs/synapse/devguides/devops_axon.rst new file mode 100644 index 0000000000..6ff11d3b86 --- /dev/null +++ b/docs/synapse/devguides/devops_axon.rst @@ -0,0 +1,4 @@ +Axon Operations +=============== + +TBD diff --git a/docs/synapse/devguides/devops_cell.rst b/docs/synapse/devguides/devops_cell.rst new file mode 100644 index 0000000000..21ee1eb833 --- /dev/null +++ b/docs/synapse/devguides/devops_cell.rst @@ -0,0 +1,90 @@ +Cell Operations +=============== + +As detailed in :ref:`dev_architecture`, the ``Cell`` implements a number of core management functionalities, +and is therefore used as the base class for Synapse applications. It is also recommended to implement ``Cell`` for +custom services. + +.. _devops-cell-config: + +Configuring a Cell Service +-------------------------- + +A Cell has a set of base configuration data that will also be inherited by any implementations: + +- ``dirn``: The storage directory for the Cell which is a required argument for service startup. +- ``telepath``: Optional override for the Telepath URL to listen on. +- ``https``: Optional override for the port to bind for the HTTPS/REST API. +- ``name``: Optional additional name to share the service as. + +The Cell class also specifies configuration variables in ``confdefs``: + +- ``auth:passwd``: Optional bootstrapping for the root user password. +- ``nexslog:en``: Optional enablement of Nexus logging (most custom Cell implementations will not set this). + +Cell implementations can extend the configuration variables available by specifying them in +``confdefs`` in the Cell subclass. The variable names can be namespaced using colons, for example ``mysvc:apikey``. + +Depending on deployment requirements, a combination of methods can be used for loading the configurations into the Cell. + +Config File +*********** + +A Cell has one optional configuration file, ``cell.yaml``, that may be located in the root Cell directory. +The format of this file is YAML, and variable names are specified without alteration, for example:: + + --- + auth:passwd: secret + mysvc:apikey: 720a50f9-cfa1-43a9-9eca-dda379ecd8c5 + ... + +Environment Variables +********************* + +Environment variable names are automatically generated for a Cell service using the following naming convention: +``SYN__``. Variable names with colons are replaced with underscores, +and the raw environment variable value is deserialized as yaml, prior to performing type validation. + +Command Line +************ + +Variables which can only be easily passed as command line arguments are available on the command line. +Variable names with colons are replaced with a single dash. + +.. note:: + + When used to launch a Cell as the entry point of a program, the configuration precedence order is: + + #. Command line arguments + #. Environment variables + #. cell.yaml values + + These may all be mixed and matched for a given deployment. + If a backup of a service is made and the deployment uses configuration data from command line arguments and + environment variables, those will need to be considered when moving/restoring the backup. + +Starting a Cell Service +----------------------- + +The examples provided below are intended for Cell implementations outside of the Synapse level components, +which have their own servers in the ``synapse.servers`` module. + +As Main Module +************** + +Cell implementations may define the following as the main application entrypoint (where MySvc is the Cell subclass):: + + if __name__ == '__main__': + asyncio.run(MySvc.execmain(sys.argv[1:])) + +The service can then be started with:: + + python -m path.to.main /path/to/dirn + +As Cell Server +************** + +The generic Cell server can also be used for starting the service by specifying the constructor as an argument:: + + python -m synapse.servers.cell path.to.MySvc /path/to/dirn + diff --git a/docs/synapse/devguides/devops_cortex.rst b/docs/synapse/devguides/devops_cortex.rst new file mode 100644 index 0000000000..f754c5fb90 --- /dev/null +++ b/docs/synapse/devguides/devops_cortex.rst @@ -0,0 +1,92 @@ +Cortex Operations +================= + +Docker Deployment +----------------- + +Configuring A Mirror +-------------------- + +.. _200_migration: + +0.1.x to 2.x.x Migration +------------------------ + +Two tools have been created to execute migration of an ``0.1.x`` Cortex to ``2.x.x``: + +* ``migrate_200`` migrates all data from the source to a new destination ``2.x.x`` Cortex. +* ``sync_200`` allows for a backup to be migrated and then synchronized with a running Cortex to facilitate minimal downtime. + +Migration Quickstart +******************** + +.. note:: + + The duration of migration is proportional to the amount of data stored, and is highly dependent on + the available system resources (especially disk I/O). For larger Cortexes it is recommended to + run migration on hardware without other highly active processes. + +#. Update the source to the latest Synapse ``0.1.x`` release. +#. Create a backup of the ``0.1.x`` Cortex. +#. In a new location install Synapse ``2.x.x`` and copy any custom modules / configurations present in the ``0.1.x`` environment. +#. Start migration using the backup as the source:: + + python -m synapse.tools.migrate_200 --src --dest + +#. Inspect the migration output for any errors that may require action (see :ref:`migration-errors` for details). +#. Startup the ``2.x.x`` Cortex. + +Migration Options +***************** + +* ``--from-last`` restarts node migration from the last checkpoint (automatically saved at periodic intervals). +* ``--safety-off`` disables form value normalization checks as a pre-condition to migrate nodes (may allow migration to run faster). +* ``--src-dedicated`` opens the source layer slabs with locked memory (must have sufficient memory available). + +Additional options that, if specified, will *not* run a migration process: + +* ``--form-counts`` is a helper utility that scans all nodes and produces counts by form for source and destination. +* ``--dump-errors`` saves migration errors to a file in the migration directory in msgpack format. + +.. _migration-errors: + +Migration Errors +**************** + +During node migration the following errors may occur, all of which indicate that the node was not migrated: + +``Unable to determine stortype`` or ``Buid/norming exception: NoSuchForm`` + +* A custom form/property may not have been properly loaded into the ``2.x.x`` environment. +* The node may not have been properly updated in a prior ``0.1.x`` datamodel migration and therefore no longer exists. + +``Normed form val does not match inbound`` or ``Calculated buid does not match inbound`` + +* Is likely due to a node that was not properly re-normalized after a prior Synapse update. + +Post-migration Synchronization +****************************** + +After migration, the ``sync_200`` service can be used to push post-backup changes to the migrated ``2.x.x`` Cortex, +and keep it updated until cut-over. ``sync_200`` uses splices to translate the changes, and therefore they must +be enabled on the source Cortex. In order to control and monitor synchronization, ``sync_200`` can be added as a Storm service. + +When synchronization is started the service will enable "migration mode" on the destination ``2.x.x`` Cortex, which +prevents cron jobs and triggers from running. Migration mode will then be disabled when the synchronization is +stopped or when the Cortex is restarted. + +#. Complete migration, including starting up the ``2.x.x`` Cortex. +#. Locate the saved splice offset file from migration at ``/migration/lyroffs.yaml``. +#. Start the ``sync_200`` service (shown with the optional ``--auth-passwd`` to bootstrap the root user):: + + python -m synapse.tools.sync_200 \ + --auth-passwd secret --offsfile \ + --src <01x_telepath_url> --dest <20x_telepath_url> + +#. Add the Storm service to the Cortex and use the available commands to start synchronization. +#. When ready to cut-over, and the read status is up-to-date, stop the synchronization using the ``stopsync`` command. + +Cleanup +******* + +After migration is fully complete, delete the now-unused directory "migration" inside the cortex directory. diff --git a/docs/synapse/devguides/devops_cryotank.rst b/docs/synapse/devguides/devops_cryotank.rst new file mode 100644 index 0000000000..3632682943 --- /dev/null +++ b/docs/synapse/devguides/devops_cryotank.rst @@ -0,0 +1,4 @@ +Cryotank Operations +=================== + +TDB \ No newline at end of file diff --git a/docs/synapse/devguides/devops_general.rst b/docs/synapse/devguides/devops_general.rst new file mode 100644 index 0000000000..408b419494 --- /dev/null +++ b/docs/synapse/devguides/devops_general.rst @@ -0,0 +1,106 @@ +General Devops +============== + +Backups +------- + +It is strongly recommended that users schedule regular backups of all services deployed within their **Synapse** +ecosystem. Each service can be backed up using the **Synapse** backup tool: ``synapse.tools.backup``. + +The **Synapse** service architecture is designed to contain everything a service needs within the directory you +specify during service startup. Take, for example, a **Cortex** started with:: + python -m synapse.servers.cortex /data/cortex00 + +The **Cortex** will be completely contained within the service working directory ``/data/cortex00``. The synapse tool +``synapse.tools.backup`` may be used to create a backup copy of this working directory which may then be restored. + +It is important that you use ``synapse.tools.backup`` rather than simply copying the directory. + +It is also worth noting that the newly created backup is a defragmented / optimized copy of all database data +structures. As such, we recommend occasionally scheduling a maintenance window to create a "cold backup" with the +service offline and deploy the backup copy when bringing the service back online. Regularly performing this +"restore from cold backup" procedure can dramatically improve performance and resource utilization. + +Running A Backup +**************** + +Continuing our previous example, running a backup is as simple as:: + python -m synapse.tools.backup /data/cortex00 /backups/cortex00_`date +%Y%m%d` + +Assuming that your backup was run on ``May 19, 2020``, this would create a backup in the directory ``/backups/cortex00_20200519``. + +The backup command can be run on a live service. Depending on your configuration, creating a live backup +of your service can temporarily degrade performance of the running service. As such, it may be best to schedule +backups during low utilization time windows. + +Restoring From Backup +********************* + +In the event that restoring from backup is necessary, simply move the service working directory and +copy a previous backup directory to the service working directory location. From our previous example, +this would involve running the following shell commands:: + mv /data/cortex00 /data/cortex00_old + cp -R /backups/cortex00_20200519 /data/cortex00 + +TLS/SSL Deployments +------------------- + +For production deployments, it is recommended that all services use the built-in ``telepath`` SSL/TLS +protocol. You may deploy a service using TLS encryption by specifying a ``--telepath`` listen URL option, such +as ``ssl://cortex.vertex.link/``. + +Under some circumstances, such as inbound DNAT networks or multi-homed hosts, it may be necessary to specify a +socket bind address that is different than the hostname. If your environment requires you to bind an address that +is different than the hostname's DNS entry, an explicit hostname query parameter may be +specified:``ssl://0.0.0.0/?hostname=cortex.vertex.link``. + +The client will also need to specify an SSL/TLS ``telepath`` URL such as: ``ssl://visi:passwd@cortex.vertex.link``. + +Once the ``ssl`` protocol is specified, the ``hostname``, either from a query parameter or from the URL's +network location, is used to lookup a matching ``crt`` and ``key`` file pair from your server certificate directory +located at ``~/.syn/certs/hosts``. See the following sections for how to setup server certificates. + +Self-Signed Certificates using certtool +*************************************** + +For self-signed certificates, we need to generate a CA certificate and key as well as a server certificate and key. + +The synapse ``certtool`` can be used to easily generate a CA and server certificates. For example, if we wanted +to generate a CA certificate for "vertex":: + + python -m synapse.tools.easycert --ca vertex + key saved: /home/visi/.syn/certs/cas/vertex.key + cert saved: /home/visi/.syn/certs/cas/vertex.crt + +We can then generate a server certificate and keyfile pair and sign it with our new CA:: + + python -m synapse.tools.easycert --server cortex.vertex.link --signas vertex + key saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.key + cert saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.crt + +To verify the server certificate, clients will need to have the ``~/.syn/certs/cas/vertex.crt`` file in their +certificate directory. + +NOTE: do not distribute the ``~/.syn/certs/cas/vertex.key`` file as that would allow regular users the ability +to sign arbitrary certificates). + +CA-Signed Certificates +********************** + +Any external CA may be used to sign ``telepath`` server certificates. The ``certtool`` can be used to easily +generate a certificate signing request (CSR) to be signed by an external CA or you can simply copy or link +pre-existing PEM encoded certificate files to the expected filesystem locations. + +To generate a CSR using ``certtool``:: + python -m synapse.tools.easycert --csr --server cortex.vertex.link + key saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.key + csr saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.csr + +You may then submit your CSR file (in this case ``~/.syn/certs/hosts/cortex.vertex.link.csr``) to your CA of choice for signing. +Once your CA returns a signed certificate in PEM format, place it in the expected location (``~/.syn/certs/hosts/cortex.vertex.link.crt`` in this example) +and it will be loaded when you start your service. + +Client-Side Certificates for Authentication +******************************************* + +TODO diff --git a/docs/synapse/devguides/stormservices.rst b/docs/synapse/devguides/stormservices.rst index bd9d677666..180e89f4a7 100644 --- a/docs/synapse/devguides/stormservices.rst +++ b/docs/synapse/devguides/stormservices.rst @@ -3,21 +3,69 @@ Storm Service Development ######################### -NOTE: These docs are an initial place holder to hold some notes. - Anatomy of a Storm Service ========================== -Storm Service Modules -===================== +A Storm Service (see :ref:`gloss-service`) is a standalone application that extends the capabilities of the Cortex. +One common use case for creating a service is to add a Storm command that will query third-party data, +translate the results into the Synapse datamodel, and then ingest them into the hypergraph. + +In order to leverage core functionalities it is recommended that Storm services are created as Cell implementations, +and the documentation that follows will assume this. For additional information see :ref:`dev_architecture`. + +A Storm service generally implements the following components: + +- A :ref:`gloss-package` that contains the new :ref:`stormservice-cmd` and optional new :ref:`stormservice-mod`. + +- A subclass of ``synapse.lib.CellApi`` which uses the ``synapse.lib.StormSvc`` mixin and contains the following information: + + - The service name, version, packages, and events as defined in ``synapse.lib.StormSvc``. + - Custom methods which will be accessible as Telepath API endpoints, and therefore available for use within defined Storm commands. + +- A subclass of ``synapse.lib.Cell`` which includes additional configuration defintions and methods required to implement the service. + +Connecting a service +-------------------- + +For instructions on configuring and starting a Cell service see :ref:`devops-cell-config`. + +Before connecting a service to a Cortex it is best practice to add a new user to the service Cell (see :ref:`initial-roles`). + +A Storm command can then be run on the Cortex to add the new service:: + + service.add mysvc tcp://:@:27495 + +Permissions to access the service can be granted by adding the ``service.get.`` rule to the appropriate users / roles. + +The new Storm commands will now be available for use, and are included in Storm ``help``. + +.. _stormservice-cmd: Storm Service Commands ====================== +Implementation +-------------- + +Multiple Storm commands can be added to a Storm service package, with each defining the following attributes: + + - ``name``: Name of the Storm command to expose in the Cortex. + - ``descr``: Description of the command which will be available in ``help`` displays. + - ``cmdargs``: An optional list of arguments for the command. + - ``cmdconf``: An optional dictionary of additional configuration variables to provide to the command Storm execution. + - ``forms``: List of input and output forms for the command. + - ``storm``: The Storm code, as a string, that will be executed when the command is called. + +Typically, the Storm code will start by getting a reference to the service via ``$svc = $lib.service.get($cmdconf.svciden)`` +and reading in any defined ``cmdargs`` that are available in ``$cmdopts``. The methods defined in the service's Cell API +can then be called by, for example, ``$retn = $svc.mysvcmethod($cmdopts.query)``. + Input/Output Conventions ------------------------ -Most commands that enrich or add additional context to nodes should simply yield the nodes they were given as inputs. If they don’t know how to enrich or add additional context to a given form, nodes of that form should be yielded rather than producing an error. This allows a series of enrichment commands to be pipelined regardless of the different inputs that a given command knows how to operate on. +Most commands that enrich or add additional context to nodes should simply yield the nodes they were given as inputs. +If they don’t know how to enrich or add additional context to a given form, nodes of that form should be yielded rather than producing an error. +This allows a series of enrichment commands to be pipelined regardless of the different inputs that a given command knows how to operate on. Argument Conventions -------------------- @@ -25,16 +73,36 @@ Argument Conventions ``--verbose`` ~~~~~~~~~~~~~ -In general, storm commands should operate silently over their input nodes and should especially avoid printing anything "per node". However, when an error occurs, the command may use ``$lib.warn()`` to print a warning message per-node. Commands should implement a ``--verbose`` command line option to enable printing "per node" informational output. +In general, Storm commands should operate silently over their input nodes and should especially avoid printing anything "per node". +However, when an error occurs, the command may use ``$lib.warn()`` to print a warning message per-node. +Commands should implement a ``--verbose`` command line option to enable printing "per node" informational output. ``--debug`` ~~~~~~~~~~~ +For commands where additional messaging would assist in debugging a ``--debug`` command line option should be implemented. +For example, a Storm command that is querying a third-party data source could use ``$lib.print()`` to print the raw query string +and raw response when the ``--debug`` option is specified. + ``--yield`` ~~~~~~~~~~~ -For commands that create additional nodes, it may be beneficial to add a --yield option to allow a query to operate on the newly created nodes. Some guidelines for ``--yield`` options: +For commands that create additional nodes, it may be beneficial to add a ``--yield`` option to allow a query to operate on the newly created nodes. +Some guidelines for ``--yield`` options: -- The command should *not* yield the input node(s) when a --yield is specified -- The --yield option should *not* be implemented when pivoting from the input node to reach the newly created node is a “refs out” or 1-to-1 direct pivot. For example, there is no need to have a --yield option on the ``maxmind`` command even though it may create an ``inet:asn`` node for an input ``inet:ipv4`` node due to the 1-to-1 pivot ``-> inet:asn`` being possible. +- The command should *not* yield the input node(s) when a ``--yield`` is specified +- The ``--yield`` option should *not* be implemented when pivoting from the input node to reach the newly created node is a “refs out” or 1-to-1 direct pivot. For example, there is no need to have a ``--yield`` option on the ``maxmind`` command even though it may create an ``inet:asn`` node for an input ``inet:ipv4`` node due to the 1-to-1 pivot ``-> inet:asn`` being possible. - The ``--yield`` option should ideally determine a “primary” node form to yield even when the command may create many forms in order to tag them or update .seen times. + +.. _stormservice-mod: + +Storm Service Modules +===================== + +Modules can be added to a Storm service package to expose reusable Storm functions. +Each module defines a ``name``, which is used for importing elsewhere via ``$lib.import()``, +and a ``storm`` string. The Storm code in this case contains callable functions with the format:: + + function myfunc(var1, var2) { + // function Storm code + } diff --git a/docs/synapse/devops.rst b/docs/synapse/devops.rst index be76e29658..9fcf030561 100644 --- a/docs/synapse/devops.rst +++ b/docs/synapse/devops.rst @@ -6,210 +6,21 @@ Synapse DevOps Guide #################### -General -======= +The DevOps guide contains information useful for running **Synapse** ecosystem components. This will continue to be +updated and expanded over time. -Backups -------- - -It is strongly recommended that users schedule regular backups of all services deployed within their **Synapse** -ecosystem. Each service can be backed up using the **Synapse** backup tool: ``synapse.tools.backup``. - -The **Synapse** service architecture is designed to contain everything a service needs within the directory you -specify during service startup. Take, for example, a **Cortex** started with:: - python -m synapse.servers.cortex /data/cortex00 - -The **Cortex** will be completely contained within the service working directory ``/data/cortex00``. The synapse tool -``synapse.tools.backup`` may be used to create a backup copy of this working directory which may then be restored. - -It is important that you use ``synapse.tools.backup`` rather than simply copying the directory. - -It is also worth noting that the newly created backup is a defragmented / optimized copy of all database data -structures. As such, we recommend occasionally scheduling a maintenance window to create a "cold backup" with the -service offline and deploy the backup copy when bringing the service back online. Regularly performing this -"restore from cold backup" procedure can dramatically improve performance and resource utilization. - -Running A Backup -**************** - -Continuing our previous example, running a backup is as simple as:: - python -m synapse.tools.backup /data/cortex00 /backups/cortex00_`date +%Y%m%d` - -Assuming that your backup was run on ``May 19, 2020``, this would create a backup in the directory ``/backups/cortex00_20200519``. - -The backup command can be run on a live service. Depending on your configuration, creating a live backup -of your service can temporarily degrade performance of the running service. As such, it may be best to schedule -backups during low utilization time windows. - -Restoring From Backup -********************* - -In the event that restoring from backup is necessary, simply move the service working directory and -copy a previous backup directory to the service working directory location. From our previous example, -this would involve running the following shell commands:: - mv /data/cortex00 /data/cortex00_old - cp -R /backups/cortex00_20200519 /data/cortex00 - -TLS/SSL Deployments -------------------- - -For production deployments, it is recommended that all services use the built-in ``telepath`` SSL/TLS -protocol. You may deploy a service using TLS encryption by specifying a ``--telepath`` listen URL option, such -as ``ssl://cortex.vertex.link/``. - -Under some circumstances, such as inbound DNAT networks or multi-homed hosts, it may be necessary to specify a -socket bind address that is different than the hostname. If your environment requires you to bind an address that -is different than the hostname's DNS entry, an explicit hostname query parameter may be -specified:``ssl://0.0.0.0/?hostname=cortex.vertex.link``. - -The client will also need to specify an SSL/TLS ``telepath`` URL such as: ``ssl://visi:passwd@cortex.vertex.link``. - -Once the ``ssl`` protocol is specified, the ``hostname``, either from a query parameter or from the URL's -network location, is used to lookup a matching ``crt`` and ``key`` file pair from your server certificate directory -located at ``~/.syn/certs/hosts``. See the following sections for how to setup server certificates. - -Self-Signed Certificates using certtool -*************************************** - -For self-signed certificates, we need to generate a CA certificate and key as well as a server certificate and key. - -The synapse ``certtool`` can be used to easily generate a CA and server certificates. For example, if we wanted -to generate a CA certificate for "vertex":: - - python -m synapse.tools.easycert --ca vertex - key saved: /home/visi/.syn/certs/cas/vertex.key - cert saved: /home/visi/.syn/certs/cas/vertex.crt - -We can then generate a server certificate and keyfile pair and sign it with our new CA:: - - python -m synapse.tools.easycert --server cortex.vertex.link --signas vertex - key saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.key - cert saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.crt - -To verify the server certificate, clients will need to have the ``~/.syn/certs/cas/vertex.crt`` file in their -certificate directory. - -NOTE: do not distribute the ``~/.syn/certs/cas/vertex.key`` file as that would allow regular users the ability -to sign arbitrary certificates). - -CA-Signed Certificates -********************** - -Any external CA may be used to sign ``telepath`` server certificates. The ``certtool`` can be used to easily -generate a certificate signing request (CSR) to be signed by an external CA or you can simply copy or link -pre-existing PEM encoded certificate files to the expected filesystem locations. - -To generate a CSR using ``certtool``:: - python -m synapse.tools.easycert --csr --server cortex.vertex.link - key saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.key - csr saved: /home/visi/.syn/certs/hosts/cortex.vertex.link.csr - -You may then submit your CSR file (in this case ``~/.syn/certs/hosts/cortex.vertex.link.csr``) to your CA of choice for signing. -Once your CA returns a signed certificate in PEM format, place it in the expected location (``~/.syn/certs/hosts/cortex.vertex.link.crt`` in this example) -and it will be loaded when you start your service. - -Client-Side Certificates for Authentication -******************************************* - -TODO - -Cortex -====== - -Docker Deployment ------------------ - -Configuring A Mirror --------------------- - -.. _200_migration: - -0.1.x to 2.x.x Migration ------------------------- - -Two tools have been created to execute migration of an ``0.1.x`` Cortex to ``2.x.x``: - -* ``migrate_200`` migrates all data from the source to a new destination ``2.x.x`` Cortex. -* ``sync_200`` allows for a backup to be migrated and then synchronized with a running Cortex to facilitate minimal downtime. - -Migration Quickstart -******************** - -.. note:: - - The duration of migration is proportional to the amount of data stored, and is highly dependent on - the available system resources (especially disk I/O). For larger Cortexes it is recommended to - run migration on hardware without other highly active processes. - -#. Update the source to the latest Synapse ``0.1.x`` release. -#. Create a backup of the ``0.1.x`` Cortex. -#. In a new location install Synapse ``2.x.x`` and copy any custom modules / configurations present in the ``0.1.x`` environment. -#. Start migration using the backup as the source:: - - python -m synapse.tools.migrate_200 --src --dest - -#. Inspect the migration output for any errors that may require action (see :ref:`migration-errors` for details). -#. Startup the ``2.x.x`` Cortex. - -Migration Options -***************** - -* ``--from-last`` restarts node migration from the last checkpoint (automatically saved at periodic intervals). -* ``--safety-off`` disables form value normalization checks as a pre-condition to migrate nodes (may allow migration to run faster). -* ``--src-dedicated`` opens the source layer slabs with locked memory (must have sufficient memory available). - -Additional options that, if specified, will *not* run a migration process: - -* ``--form-counts`` is a helper utility that scans all nodes and produces counts by form for source and destination. -* ``--dump-errors`` saves migration errors to a file in the migration directory in msgpack format. - -.. _migration-errors: - -Migration Errors -**************** - -During node migration the following errors may occur, all of which indicate that the node was not migrated: - -``Unable to determine stortype`` or ``Buid/norming exception: NoSuchForm`` - -* A custom form/property may not have been properly loaded into the ``2.x.x`` environment. -* The node may not have been properly updated in a prior ``0.1.x`` datamodel migration and therefore no longer exists. - -``Normed form val does not match inbound`` or ``Calculated buid does not match inbound`` - -* Is likely due to a node that was not properly re-normalized after a prior Synapse update. - -Post-migration Synchronization -****************************** - -After migration, the ``sync_200`` service can be used to push post-backup changes to the migrated ``2.x.x`` Cortex, -and keep it updated until cut-over. ``sync_200`` uses splices to translate the changes, and therefore they must -be enabled on the source Cortex. In order to control and monitor synchronization, ``sync_200`` can be added as a Storm service. - -When synchronization is started the service will enable "migration mode" on the destination ``2.x.x`` Cortex, which -prevents cron jobs and triggers from running. Migration mode will then be disabled when the synchronization is -stopped or when the Cortex is restarted. - -#. Complete migration, including starting up the ``2.x.x`` Cortex. -#. Locate the saved splice offset file from migration at ``/migration/lyroffs.yaml``. -#. Start the ``sync_200`` service (shown with the optional ``--auth-passwd`` to bootstrap the root user):: - - python -m synapse.tools.sync_200 \ - --auth-passwd secret --offsfile \ - --src <01x_telepath_url> --dest <20x_telepath_url> - -#. Add the Storm service to the Cortex and use the available commands to start synchronization. -#. When ready to cut-over, and the read status is up-to-date, stop the synchronization using the ``stopsync`` command. - -Cleanup -******* - -After migration is fully complete, delete the now-unused directory "migration" inside the cortex directory. +.. toctree:: + :titlesonly: -Axon -==== + devguides/devops_general + devguides/devops_cell + devguides/devops_cortex + devguides/devops_axon + devguides/devops_cryotank -Cryotank -======== + autodocs/conf_axon + autodocs/conf_cortex + autodocs/conf_cryocell + autodocs/conf_syncmigrator .. _index: ../index.html diff --git a/synapse/cortex.py b/synapse/cortex.py index 6d11c0d970..0a38e0979b 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -702,7 +702,7 @@ class Cortex(s_cell.Cell): # type: ignore 'type': 'string' }, 'mirror': { - 'description': 'Run a mirror of the cortex at the given telepath URL. (we must be a backup!)', + 'description': 'Run a mirror of the cortex at the given telepath URL. We must be a backup!', 'type': 'string' }, 'cron:enable': { @@ -732,7 +732,7 @@ class Cortex(s_cell.Cell): # type: ignore }, 'provenance:en': { 'default': False, - 'description': 'Enable provenance tracking for all writes', + 'description': 'Enable provenance tracking for all writes.', 'type': 'boolean' }, 'modules': { diff --git a/synapse/lib/config.py b/synapse/lib/config.py index e07ad46a99..22ec2e5719 100644 --- a/synapse/lib/config.py +++ b/synapse/lib/config.py @@ -137,6 +137,7 @@ def __init__(self, conf = {} self.conf = conf self._argparse_conf_names = {} + self._argparse_conf_parsed_names = {} self.envar_prefix = envar_prefix self.validator = getJsValidator(self.json_schema) @@ -183,11 +184,18 @@ def getArgParseArgs(self): parsed_name = name.replace(':', '-') replace_name = name.replace(':', '_') self._argparse_conf_names[replace_name] = name + self._argparse_conf_parsed_names[name] = parsed_name argname = '--' + parsed_name argdata.append((argname, akwargs)) return argdata + def getCmdlineMapping(self): + if not self._argparse_conf_parsed_names: + # Giv a shot at populating the data + _ = self.getArgParseArgs() + return {k: v for k, v in self._argparse_conf_parsed_names.items()} + def setConfFromOpts(self, opts): ''' Set the opts for a conf object from a namespace object. @@ -234,11 +242,8 @@ def setConfFromEnvs(self): Returns: None: Returns None. ''' - for name, conf in self.json_schema.get('properties', {}).items(): - if conf.get('hideconf'): - continue - - envar = make_envar_name(name, prefix=self.envar_prefix) + name2envar = self.getEnvarMapping() + for name, envar in name2envar.items(): envv = os.getenv(envar) if envv is not None: envv = yaml.safe_load(envv) @@ -246,6 +251,22 @@ def setConfFromEnvs(self): if resp == envv: logger.debug(f'Set config valu from envar: [{envar}]') + def getEnvarMapping(self): + ''' + Get a mapping of config values to envars. + + Configuration values which have the ``hideconf`` value set to True are not resolved from environment + variables. + ''' + ret = {} + for name, conf in self.json_schema.get('properties', {}).items(): + if conf.get('hideconf'): + continue + + envar = make_envar_name(name, prefix=self.envar_prefix) + ret[name] = envar + return ret + # General methods def reqConfValid(self): ''' diff --git a/synapse/tests/test_lib_stormsvc.py b/synapse/tests/test_lib_stormsvc.py index 959c28b5fd..9ba3a3c1bd 100644 --- a/synapse/tests/test_lib_stormsvc.py +++ b/synapse/tests/test_lib_stormsvc.py @@ -245,7 +245,7 @@ class LifterService(s_stormsvc.StormSvc): 'commands': ( { 'name': 'lifter', - 'desc': 'Lift inet:ipv4=1.2.3.4', + 'descr': 'Lift inet:ipv4=1.2.3.4', 'storm': 'inet:ipv4=1.2.3.4', }, ), @@ -269,10 +269,15 @@ class StormvarService(s_cell.CellApi, s_stormsvc.StormSvc): 'commands': ( { 'name': 'magic', + 'descr': 'Test stormvar support.', 'cmdargs': ( ('name', {}), ('--debug', {'default': False, 'action': 'store_true'}) ), + 'forms': { + 'input': ('test:str', 'test:int'), + 'output': ('test:comp', 'inet:ipv4'), + }, 'storm': ''' $fooz = $cmdopts.name if $cmdopts.debug { diff --git a/synapse/tests/test_tools_autodoc.py b/synapse/tests/test_tools_autodoc.py index ab78e67674..bf0089ab14 100644 --- a/synapse/tests/test_tools_autodoc.py +++ b/synapse/tests/test_tools_autodoc.py @@ -34,3 +34,63 @@ async def test_tools_autodoc_docmodel(self): self.isin('Universal props are system level properties which may be present on every node.', s) self.isin('.created', s) self.notin('..created\n', s) + + async def test_tools_autodoc_confdefs(self): + + with self.getTestDir() as path: + + argv = ['--savedir', path, '--doc-conf', + 'synapse.tests.test_lib_stormsvc.StormvarServiceCell'] + + outp = self.getTestOutp() + self.eq(await s_autodoc.main(argv, outp=outp), 0) + + with s_common.genfile(path, 'conf_stormvarservicecell.rst') as fd: + buf = fd.read() + s = buf.decode() + + self.isin('StormvarServiceCell Configuration Options', s) + self.isin('See :ref:`devops-cell-config` for', s) + self.isin('auth\:passwd', s) + self.isin('Environment Variable\n ``SYN_STORMVARSERVICECELL_AUTH_PASSWD``', s) + self.isin('``--auth-passwd``', s) + + argv.append('--doc-conf-reflink') + argv.append('`Configuring a Cell Service `_') + + # truncate the current file + with s_common.genfile(path, 'conf_stormvarservicecell.rst') as fd: + fd.truncate() + + outp = self.getTestOutp() + self.eq(await s_autodoc.main(argv, outp=outp), 0) + with s_common.genfile(path, 'conf_stormvarservicecell.rst') as fd: + buf = fd.read() + s = buf.decode() + + self.isin('StormvarServiceCell Configuration Options', s) + self.isin('See `Configuring a Cell Service