diff --git a/Dockerfile b/Dockerfile index f7552f4c48..41706b0f1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,3 +15,5 @@ COPY synapse/docker/start-cron.sh /start-cron.sh COPY docker/bootstrap.sh /build/synapse/bootstrap.sh RUN /build/synapse/bootstrap.sh + +VOLUME /vertex/storage diff --git a/docs/synapse/devguides/devops_axon.rst b/docs/synapse/devguides/devops_axon.rst index 6ff11d3b86..4a4e64fc86 100644 --- a/docs/synapse/devguides/devops_axon.rst +++ b/docs/synapse/devguides/devops_axon.rst @@ -1,4 +1,10 @@ Axon Operations =============== -TBD +The Axon is an interface for providing binary / blob storage inside of the Synapse ecosystem. This indexes binaries +based on SHA-256 hash so we do not duplicate the storage of the same set of bytes twice. + +Configuration Options +--------------------- + +For a list of boot time configuration options for the Axon, see the listing at :ref:`autodoc-axon-conf`. diff --git a/docs/synapse/devguides/devops_cell.rst b/docs/synapse/devguides/devops_cell.rst index 21ee1eb833..2900cd9b6c 100644 --- a/docs/synapse/devguides/devops_cell.rst +++ b/docs/synapse/devguides/devops_cell.rst @@ -3,19 +3,19 @@ Cell Operations As detailed in :ref:`dev_architecture`, the ``Cell`` implements a number of core management functionalities, and is therefore used as the base class for Synapse applications. It is also recommended to implement ``Cell`` for -custom services. +custom Storm Services and other Synapse related components. .. _devops-cell-config: -Configuring a Cell Service --------------------------- +Configuring a Cell +------------------ A Cell has a set of base configuration data that will also be inherited by any implementations: -- ``dirn``: The storage directory for the Cell which is a required argument for service startup. +- ``dirn``: The storage directory for the Cell which is a required argument for Cell startup. - ``telepath``: Optional override for the Telepath URL to listen on. - ``https``: Optional override for the port to bind for the HTTPS/REST API. -- ``name``: Optional additional name to share the service as. +- ``name``: Optional additional name to share the cell as. The Cell class also specifies configuration variables in ``confdefs``: @@ -27,6 +27,18 @@ Cell implementations can extend the configuration variables available by specify Depending on deployment requirements, a combination of methods can be used for loading the configurations into the Cell. +.. note:: + The Cell directory (refered to as ``dirn``) should be considered a persistent directory for a given Synapse + cell. Inside of this directory there are several files stored which are necessary in order for a given instance + of a cell deployment to work properly. + + Docker images made by Vertex to support Synapse cells will have default volumes for ``/vertex/storage``. + We use this as the default cell directory for default entry points in documentation. This location can either + have a persistent docker volume present for it created, or a external location on disk can be mapped into this + location. Any orchestration tooling should consider the requirements for cell directory data to be persistent, + unless stated otherwise. + + Config File *********** @@ -41,8 +53,8 @@ The format of this file is YAML, and variable names are specified without altera Environment Variables ********************* -Environment variable names are automatically generated for a Cell service using the following naming convention: -``SYN__``. Variable names with colons are replaced with underscores, +Environment variable names are automatically generated for a Cell configuration options using the following naming +convention: ``SYN__``. Variable names with colons are replaced with underscores, and the raw environment variable value is deserialized as yaml, prior to performing type validation. Command Line @@ -60,11 +72,11 @@ Variable names with colons are replaced with a single dash. #. cell.yaml values These may all be mixed and matched for a given deployment. - If a backup of a service is made and the deployment uses configuration data from command line arguments and + If a backup of a Cell is made and the deployment uses configuration data from command line arguments and environment variables, those will need to be considered when moving/restoring the backup. -Starting a Cell Service ------------------------ +Starting a Cell +--------------- The examples provided below are intended for Cell implementations outside of the Synapse level components, which have their own servers in the ``synapse.servers`` module. @@ -72,19 +84,19 @@ which have their own servers in the ``synapse.servers`` module. As Main Module ************** -Cell implementations may define the following as the main application entrypoint (where MySvc is the Cell subclass):: +Cell implementations may define the following as the main application entrypoint (where MyCell is the Cell subclass):: if __name__ == '__main__': - asyncio.run(MySvc.execmain(sys.argv[1:])) + asyncio.run(MyCell.execmain(sys.argv[1:])) -The service can then be started with:: +The cell can then be started with:: python -m path.to.main /path/to/dirn As Cell Server ************** -The generic Cell server can also be used for starting the service by specifying the constructor as an argument:: +The generic Cell server can also be used for starting the Cell by specifying the constructor as an argument:: - python -m synapse.servers.cell path.to.MySvc /path/to/dirn + python -m synapse.servers.cell path.to.MyCell /path/to/dirn diff --git a/docs/synapse/devguides/devops_cortex.rst b/docs/synapse/devguides/devops_cortex.rst index f754c5fb90..4cc7c6c6fd 100644 --- a/docs/synapse/devguides/devops_cortex.rst +++ b/docs/synapse/devguides/devops_cortex.rst @@ -90,3 +90,8 @@ Cleanup ******* After migration is fully complete, delete the now-unused directory "migration" inside the cortex directory. + +Configuration Options +--------------------- + +For a list of boot time configuration options for the Cortex, see the listing at :ref:`autodoc-cortex-conf`. diff --git a/docs/synapse/devguides/devops_cryotank.rst b/docs/synapse/devguides/devops_cryotank.rst index 3632682943..d264e0f701 100644 --- a/docs/synapse/devguides/devops_cryotank.rst +++ b/docs/synapse/devguides/devops_cryotank.rst @@ -1,4 +1,7 @@ Cryotank Operations =================== -TDB \ No newline at end of file +Configuration Options +--------------------- + +For a list of boot time configuration options for the Cryotank, see the listing at :ref:`autodoc-cryocell-conf`. diff --git a/docs/synapse/devguides/index_autoconf.rst b/docs/synapse/devguides/index_autoconf.rst new file mode 100644 index 0000000000..3ed6b977e5 --- /dev/null +++ b/docs/synapse/devguides/index_autoconf.rst @@ -0,0 +1,13 @@ +Cell Configuration Options +########################## + +The following section contains configuration options for Synapse Cell implementations which are baked into the core +Synapse package. These boot-time options may be set using information found at :ref:`devops-cell-config`. + +.. toctree:: + :titlesonly: + + ../autodocs/conf_axon + ../autodocs/conf_cortex + ../autodocs/conf_cryocell + ../autodocs/conf_syncmigrator diff --git a/docs/synapse/devguides/index_cells.rst b/docs/synapse/devguides/index_cells.rst new file mode 100644 index 0000000000..842181effa --- /dev/null +++ b/docs/synapse/devguides/index_cells.rst @@ -0,0 +1,15 @@ + +Synapse Devops +############## + +The DevOps guide contains information useful for running **Synapse** ecosystem components. This will continue to be +updated and expanded over time. + +.. toctree:: + :titlesonly: + + devops_general + devops_cell + devops_cortex + devops_axon + devops_cryotank diff --git a/docs/synapse/devops.rst b/docs/synapse/devops.rst index 9fcf030561..fc85363e0a 100644 --- a/docs/synapse/devops.rst +++ b/docs/synapse/devops.rst @@ -12,15 +12,7 @@ updated and expanded over time. .. toctree:: :titlesonly: - devguides/devops_general - devguides/devops_cell - devguides/devops_cortex - devguides/devops_axon - devguides/devops_cryotank - - autodocs/conf_axon - autodocs/conf_cortex - autodocs/conf_cryocell - autodocs/conf_syncmigrator + devguides/index_cells + devguides/index_autoconf .. _index: ../index.html diff --git a/synapse/tests/test_tools_autodoc.py b/synapse/tests/test_tools_autodoc.py index bf0089ab14..5051e48f5e 100644 --- a/synapse/tests/test_tools_autodoc.py +++ b/synapse/tests/test_tools_autodoc.py @@ -49,6 +49,7 @@ async def test_tools_autodoc_confdefs(self): buf = fd.read() s = buf.decode() + self.isin('autodoc-stormvarservicecell-conf', s) self.isin('StormvarServiceCell Configuration Options', s) self.isin('See :ref:`devops-cell-config` for', s) self.isin('auth\:passwd', s) diff --git a/synapse/tools/autodoc.py b/synapse/tools/autodoc.py index a46e748041..fdb12d1585 100644 --- a/synapse/tools/autodoc.py +++ b/synapse/tools/autodoc.py @@ -363,7 +363,7 @@ async def docConfdefs(ctor, reflink=':ref:`devops-cell-config`'): clsname = cls.__name__ conf = cls.initCellConf() # type: s_config.Config - rst.addHead(f'{clsname} Configuration Options', lvl=0) + rst.addHead(f'{clsname} Configuration Options', lvl=0, link=f'.. _autodoc-{clsname.lower()}-conf:') rst.addLines(f'The following are boot-time configuration options for the cell.') rst.addLines(f'See {reflink} for details on how to set these options.')