From 5fa383371ac884952d13ca91e1ab242b35e52100 Mon Sep 17 00:00:00 2001 From: jeanluc Date: Tue, 23 Apr 2024 16:20:02 +0200 Subject: [PATCH] Add `vault_db` modules (draft) The test part was migrated from Salt PR 63314 to salvage it, it's not finished and does not work as committed. --- docs/ref/modules/index.rst | 1 + .../saltext.vault.modules.vault_db.rst | 5 + docs/ref/states/index.rst | 1 + .../states/saltext.vault.states.vault_db.rst | 5 + docs/ref/utils/index.rst | 1 + .../utils/saltext.vault.utils.vault.db.rst | 5 + src/saltext/vault/modules/vault_db.py | 818 ++++++++++++++++++ src/saltext/vault/states/vault_db.py | 765 ++++++++++++++++ src/saltext/vault/utils/vault/db.py | 158 ++++ tests/functional/modules/test_vault_db.py | 369 ++++++++ tests/functional/states/test_vault_db.py | 375 ++++++++ tests/support/mysql.py | 195 +++++ tests/support/vault.py | 50 ++ 13 files changed, 2748 insertions(+) create mode 100644 docs/ref/modules/saltext.vault.modules.vault_db.rst create mode 100644 docs/ref/states/saltext.vault.states.vault_db.rst create mode 100644 docs/ref/utils/saltext.vault.utils.vault.db.rst create mode 100644 src/saltext/vault/modules/vault_db.py create mode 100644 src/saltext/vault/states/vault_db.py create mode 100644 src/saltext/vault/utils/vault/db.py create mode 100644 tests/functional/modules/test_vault_db.py create mode 100644 tests/functional/states/test_vault_db.py create mode 100644 tests/support/mysql.py diff --git a/docs/ref/modules/index.rst b/docs/ref/modules/index.rst index 29603e72..2f6ace70 100644 --- a/docs/ref/modules/index.rst +++ b/docs/ref/modules/index.rst @@ -10,3 +10,4 @@ _________________ :toctree: vault + vault_db diff --git a/docs/ref/modules/saltext.vault.modules.vault_db.rst b/docs/ref/modules/saltext.vault.modules.vault_db.rst new file mode 100644 index 00000000..5a5d8f1e --- /dev/null +++ b/docs/ref/modules/saltext.vault.modules.vault_db.rst @@ -0,0 +1,5 @@ +``vault_db`` +============ + +.. automodule:: saltext.vault.modules.vault_db + :members: diff --git a/docs/ref/states/index.rst b/docs/ref/states/index.rst index 6689ef03..eac45ce6 100644 --- a/docs/ref/states/index.rst +++ b/docs/ref/states/index.rst @@ -10,3 +10,4 @@ _____________ :toctree: vault + vault_db diff --git a/docs/ref/states/saltext.vault.states.vault_db.rst b/docs/ref/states/saltext.vault.states.vault_db.rst new file mode 100644 index 00000000..c2cdd1b5 --- /dev/null +++ b/docs/ref/states/saltext.vault.states.vault_db.rst @@ -0,0 +1,5 @@ +``vault_db`` +============ + +.. automodule:: saltext.vault.states.vault_db + :members: diff --git a/docs/ref/utils/index.rst b/docs/ref/utils/index.rst index a563f8c0..3dfbd9a3 100644 --- a/docs/ref/utils/index.rst +++ b/docs/ref/utils/index.rst @@ -14,6 +14,7 @@ _________ vault.auth vault.cache vault.client + vault.db vault.exceptions vault.factory vault.helpers diff --git a/docs/ref/utils/saltext.vault.utils.vault.db.rst b/docs/ref/utils/saltext.vault.utils.vault.db.rst new file mode 100644 index 00000000..40f018a7 --- /dev/null +++ b/docs/ref/utils/saltext.vault.utils.vault.db.rst @@ -0,0 +1,5 @@ +saltext.vault.utils.vault.db +============================ + +.. automodule:: saltext.vault.utils.vault.db + :members: diff --git a/src/saltext/vault/modules/vault_db.py b/src/saltext/vault/modules/vault_db.py new file mode 100644 index 00000000..2300a26f --- /dev/null +++ b/src/saltext/vault/modules/vault_db.py @@ -0,0 +1,818 @@ +""" +Manage the Vault database secret engine, request and cache +leased database credentials. + +.. important:: + This module requires the general :ref:`Vault setup `. +""" +import logging +from datetime import datetime +from datetime import timezone + +import saltext.vault.utils.vault as vault +import saltext.vault.utils.vault.db as vaultdb +from salt.exceptions import CommandExecutionError +from salt.exceptions import SaltInvocationError + +log = logging.getLogger(__name__) + + +def list_connections(mount="database"): + """ + List configured database connections. + + `API method docs `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.list_connections + + mount + The mount path the DB backend is mounted to. Defaults to ``database``. + """ + endpoint = f"{mount}/config" + try: + return vault.query("LIST", endpoint, __opts__, __context__)["data"]["keys"] + except vault.VaultNotFoundError: + return [] + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + +def fetch_connection(name, mount="database"): + """ + Read a configured database connection. Returns None if it does not exist. + + `API method docs `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.fetch_connection mydb + + name + The name of the database connection. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + endpoint = f"{mount}/config/{name}" + try: + return vault.query("GET", endpoint, __opts__, __context__)["data"] + except vault.VaultNotFoundError: + return None + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + +def write_connection( + name, + plugin, + version="", + verify=True, + allowed_roles=None, + root_rotation_statements=None, + password_policy=None, + rotate=True, + mount="database", + **kwargs, +): + """ + Create/update a configured database connection. + + .. note:: + + This endpoint distinguishes between create and update ACL capabilities. + + .. note:: + + It is highly recommended to use a Vault-specific user rather than the admin user in the + database when configuring the plugin. This user will be used to create/update/delete users + within the database so it will need to have the appropriate permissions to do so. + If the plugin supports rotating the root credentials, it is highly recommended to perform + that action after configuring the plugin. This will change the password of the user + configured in this step. The new password will not be viewable by users. + + `API method docs `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.write_connection mydb elasticsearch \ + url=http://127.0.0.1:9200 username=vault password=hunter2 + + name + The name of the database connection. + + plugin + The name of the database plugin. Known plugins to this module are: + ``cassandra``, ``couchbase``, ``elasticsearch``, ``influxdb``, ``hanadb``, ``mongodb``, + ``mongodb_atlas``, ``mssql``, ``mysql``, ``oracle``, ``postgresql``, ``redis``, + ``redis_elasticache``, ``redshift``, ``snowflake``. + If you pass an unknown plugin, make sure its Vault-internal name can be formatted + as ``{plugin}-database-plugin`` and to pass all required parameters as kwargs. + + version + Specifies the semantic version of the plugin to use for this connection. + + verify + Verify the connection during initial configuration. Defaults to True. + + allowed_roles + List of the roles allowed to use this connection. ``["*"]`` means any role + can use this connection. Defaults to empty (no role can use it). + + root_rotation_statements + Specifies the database statements to be executed to rotate the root user's credentials. + See the plugin's API page for more information on support and formatting for this parameter. + + password_policy + The name of the password policy to use when generating passwords for this database. + If not specified, this will use a default policy defined as: + 20 characters with at least 1 uppercase, 1 lowercase, 1 number, and 1 dash character. + + rotate + Rotate the root credentials after plugin setup. Defaults to True. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + + kwargs + Different plugins require different parameters. You need to make sure that you pass them + as supplemental keyword arguments. For known plugins, the required arguments will + be checked. + """ + endpoint = f"{mount}/config/{name}" + plugin_meta = vaultdb.get_plugin_meta(plugin) + plugin_name = plugin_meta["name"] or plugin + payload = {k: v for k, v in kwargs.items() if not k.startswith("_")} + + if fetch_connection(name, mount=mount) is None: + missing_kwargs = set(plugin_meta["required"]) - set(payload) + if missing_kwargs: + raise SaltInvocationError( + f"The plugin {plugin} requires the following additional kwargs: {missing_kwargs}." + ) + + payload["plugin_name"] = f"{plugin_name}-database-plugin" + payload["verify_connection"] = verify + if version is not None: + payload["plugin_version"] = version + if allowed_roles is not None: + payload["allowed_roles"] = allowed_roles + if root_rotation_statements is not None: + payload["root_rotation_statements"] = root_rotation_statements + if password_policy is not None: + payload["password_policy"] = password_policy + + try: + vault.query("POST", endpoint, __opts__, __context__, payload=payload) + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + if not rotate: + return True + return rotate_root(name, mount=mount) + + +def delete_connection(name, mount="database"): + """ + Delete a configured database connection. Returns None if it does not exist. + + `API method docs `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.delete_connection mydb + + name + The name of the database connection. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + endpoint = f"{mount}/config/{name}" + try: + return vault.query("DELETE", endpoint, __opts__, __context__) + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + +def reset_connection(name, mount="database"): + """ + Close a connection and restart its plugin with the configuration stored in the barrier. + + `API method docs `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.reset_connection mydb + + name + The name of the database connection. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + endpoint = f"{mount}/reset/{name}" + try: + return vault.query("POST", endpoint, __opts__, __context__) + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + +def rotate_root(name, mount="database"): + """ + Rotate the "root" user credentials stored for the database connection. + + .. warning:: + + The rotated password will not be accessible, so it is highly recommended to create + a dedicated user account as Vault's configured "root". + + `API method docs `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.rotate_root mydb + + name + The name of the database connection. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + endpoint = f"{mount}/rotate-root/{name}" + try: + return vault.query("POST", endpoint, __opts__, __context__) + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + +def list_roles(static=False, mount="database"): + """ + List configured database roles. + + `API method docs `_. + `API method docs static `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.list_roles + + static + Whether to list static roles. Defaults to False. + + mount + The mount path the DB backend is mounted to. Defaults to ``database``. + """ + endpoint = f"{mount}/{'static-' if static else ''}roles" + try: + return vault.query("LIST", endpoint, __opts__, __context__)["data"]["keys"] + except vault.VaultNotFoundError: + return [] + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + +def fetch_role(name, static=False, mount="database"): + """ + Read a configured database role. Returns None if it does not exist. + + `API method docs `_. + `API method docs static `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.fetch_role myrole + + name + The name of the database role. + + static + Whether this role is static. Defaults to False. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + endpoint = f"{mount}/{'static-' if static else ''}roles/{name}" + try: + return vault.query("GET", endpoint, __opts__, __context__)["data"] + except vault.VaultNotFoundError: + return None + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + +def write_static_role( + name, + connection, + username, + rotation_period, + rotation_statements=None, + credential_type=None, + credential_config=None, + mount="database", +): + """ + Create/update a database Static Role. Mind that not all databases support Static Roles. + + `API method docs `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.write_static_role myrole mydb myuser 24h + + name + The name of the database role. + + connection + The name of the database connection this role applies to. + + username + The username to manage. + + rotation_period + Specifies the amount of time Vault should wait before rotating the password. + The minimum is ``5s``. + + rotation_statements + Specifies the database statements to be executed to rotate the password for the + configured database user. Not every plugin type will support this functionality. + + credential_type + Specifies the type of credential that will be generated for the role. + Options include: ``password``, ``rsa_private_key``. Defaults to ``password``. + See the plugin's API page for credential types supported by individual databases. + + credential_config + Specifies the configuration for the given ``credential_type`` as a mapping. + For ``password``, only ``password_policy`` can be passed. + For ``rsa_private_key``, ``key_bits`` (defaults to 2048) and ``format`` + (defaults to ``pkcs8``) are available. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + payload = { + "username": username, + "rotation_period": rotation_period, + } + if rotation_statements is not None: + payload["rotation_statements"] = rotation_statements + return _write_role( + name, + connection, + payload, + credential_type=credential_type, + credential_config=credential_config, + static=True, + mount=mount, + ) + + +def write_role( + name, + connection, + creation_statements, + default_ttl=None, + max_ttl=None, + revocation_statements=None, + rollback_statements=None, + renew_statements=None, + credential_type=None, + credential_config=None, + mount="database", +): + r""" + Create/update a regular database role. + + `API method docs `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.write_role myrole mydb \ + \["CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}'", "GRANT SELECT ON *.* TO '{{name}}'@'%'"\] + + name + The name of the database role. + + connection + The name of the database connection this role applies to. + + creation_statements + Specifies a list of database statements executed to create and configure a user, + usually templated with {{name}} and {{password}}. Required. + + default_ttl + Specifies the TTL for the leases associated with this role. Accepts time suffixed + strings (1h) or an integer number of seconds. Defaults to system/engine default TTL time. + + max_ttl + Specifies the maximum TTL for the leases associated with this role. Accepts time suffixed + strings (1h) or an integer number of seconds. Defaults to sys/mounts's default TTL time; + this value is allowed to be less than the mount max TTL (or, if not set, + the system max TTL), but it is not allowed to be longer. + + revocation_statements + Specifies a list of database statements to be executed to revoke a user. + + rollback_statements + Specifies a list of database statements to be executed to rollback a create operation + in the event of an error. Availability and formatting depend on the specific plugin. + + renew_statements + Specifies a list of database statements to be executed to renew a user. + Availability and formatting depend on the specific plugin. + + credential_type + Specifies the type of credential that will be generated for the role. + Options include: ``password``, ``rsa_private_key``. Defaults to ``password``. + See the plugin's API page for credential types supported by individual databases. + + credential_config + Specifies the configuration for the given ``credential_type`` as a mapping. + For ``password``, only ``password_policy`` can be passed. + For ``rsa_private_key``, ``key_bits`` (defaults to 2048) and ``format`` + (defaults to ``pkcs8``) are available. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + payload = { + "creation_statements": creation_statements, + } + if default_ttl is not None: + payload["default_ttl"] = default_ttl + if max_ttl is not None: + payload["max_ttl"] = max_ttl + if revocation_statements is not None: + payload["revocation_statements"] = revocation_statements + if rollback_statements is not None: + payload["rollback_statements"] = rollback_statements + if renew_statements is not None: + payload["renew_statements"] = renew_statements + return _write_role( + name, + connection, + payload, + credential_type=credential_type, + credential_config=credential_config, + static=False, + mount=mount, + ) + + +def _write_role( + name, + connection, + payload, + credential_type=None, + credential_config=None, + static=False, + mount="database", +): + endpoint = f"{mount}/{'static-' if static else ''}roles/{name}" + payload["db_name"] = connection + if credential_type is not None: + payload["credential_type"] = credential_type + if credential_config is not None: + valid_cred_configs = { + "password": ["password_policy"], + "rsa_private_key": ["key_bits", "format"], + } + credential_type = credential_type or "password" + if credential_type in valid_cred_configs: + invalid_configs = set(credential_config) - set(valid_cred_configs[credential_type]) + if invalid_configs: + raise SaltInvocationError( + f"The following options are invalid for credential type {credential_type}: {invalid_configs}" + ) + payload["credential_config"] = credential_config + try: + return vault.query("POST", endpoint, __opts__, __context__, payload=payload) + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + +def delete_role(name, static=False, mount="database"): + """ + Delete a configured database role. + + `API method docs `_. + `API method docs static `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.delete_role myrole + + name + The name of the database role. + + static + Whether this role is static. Defaults to False. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + endpoint = f"{mount}/{'static-' if static else ''}roles/{name}" + try: + return vault.query("DELETE", endpoint, __opts__, __context__) + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + +def get_creds( + name, + static=False, + cache=True, + valid_for=None, + check_server=False, + renew_increment=None, + revoke_delay=None, + meta=None, + mount="database", + _warn_about_attr_change=True, +): + """ + Read credentials based on the named role. + + `API method docs `_. + `API method docs static `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.get_creds myrole + + name + The name of the database role. + + static + Whether this role is static. Defaults to False. + + cache + Whether to use cached credentials local to this minion to avoid + unnecessary reissuance. + When ``static`` is false, set this to a string to be able to use multiple + distinct credentials using the same role on the same minion. + Set this to false to disable caching. + Defaults to true. + + .. note:: + + This uses the same cache backend as the Vault integration, so make + sure you configure a persistent backend like ``disk`` if you expect + the credentials to survive a single run. + + + valid_for + When using cache, ensure the credentials are valid for at least this + amount of time, otherwise request new ones. + This can be an integer, which will be interpreted as seconds, or a time string + using the same format as Vault does: + Suffix ``s`` for seconds, ``m`` for minuts, ``h`` for hours, ``d`` for days. + This will be cached together with the lease and might be used by other + modules later. + + check_server + Check on the Vault server whether the lease is still active and was not + revoked early. Defaults to false. + + renew_increment + When using cache and ``valid_for`` results in a renewal attempt, request this + amount of time extension on the lease. This will be cached together with the + lease and might be used by other modules later. + + revoke_delay + When using cache and ``valid_for`` results in a revocation, set the lease + validity to this value to allow a short amount of delay between the issuance + of the new lease and the revocation of the old one. Defaults to ``60``. + This will be cached together with the lease and might be used by other + modules later. + + meta + When using cache, this value will be cached together with the lease. It will + be emitted by the ``vault_lease`` beacon module whenever a lease is + running out (usually because it cannot be extended further). It is intended + to support the reactor in deciding what needs to be done in order + to to reconfigure dependent, Vault-unaware software with newly issued + credentials. Entirely optional. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + endpoint = f"{mount}/{'static-' if static else ''}creds/{name}" + + if cache: + ckey = f"db.{mount}.{'static' if static else 'dynamic'}.{name}" + if not static and isinstance(cache, str): + ckey += f".{cache}" + else: + ckey += ".default" + creds_cache = vault.get_lease_store(__opts__, __context__) + cached_creds = creds_cache.get( + ckey, valid_for=valid_for, revoke=revoke_delay, check_server=check_server + ) + if cached_creds: + changed = False + for attr, val in ( + ("min_ttl", valid_for), + ("renew_increment", renew_increment), + ("revoke_delay", revoke_delay), + ("meta", meta), + ): + if val is not None and getattr(cached_creds, attr) != val: + setattr(cached_creds, attr, val) + changed = True + if changed: + # Warn about changes if a lease is managed by the state module + # and this function is called e.g. during YAML rendering, overwriting + # the desired attributes. The state module sets this to false. + if _warn_about_attr_change: + log.warning(f"Cached credential `{ckey}` changed lifecycle attributes") + creds_cache.store(ckey, cached_creds) + return cached_creds.data + + try: + res = vault.query("GET", endpoint, __opts__, __context__) + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err + + lease = vault.VaultLease( + min_ttl=valid_for, + renew_increment=renew_increment, + revoke_delay=revoke_delay, + meta=meta, + **res, + ) + if cache: + creds_cache.store(ckey, lease) + return lease.data + + +def clear_cached(name=None, mount=None, cache=None, static=None, delta=None, flush_on_failure=True): + """ + Clear and revoke cached database credentials matching specified parameters. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.clear_cached name=myrole mount=database + salt '*' vault_db.clear_cached mount=database + salt '*' vault_db.clear_cached + + name + Only clear credentials using this role name. + + mount + Only clear credentials from this mount. + + cache + Only clear credentials using this cache name (refer to get_creds for details). + + static + Only clear static (``True``) or dynamic (``False``) credentials. + + delta + Time after which the leases should be revoked by Vault. + Defaults to what was set on the lease(s) during creation or 60s. + + flush_on_failure + If a revocation fails, remove the lease from cache anyways. + Defaults to true. + """ + creds_cache = vault.get_lease_store(__opts__, __context__) + return creds_cache.revoke_cached( + match=vaultdb.create_cache_pattern(name=name, mount=mount, cache=cache, static=static), + delta=delta, + flush_on_failure=flush_on_failure, + ) + + +def list_cached(name=None, mount=None, cache=None, static=None): + """ + List cached database credentials matching specified parameters. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.list_cached name=myrole mount=database + salt '*' vault_db.list_cached mount=database + salt '*' vault_db.list_cached + + name + Only list credentials using this role name. + + mount + Only list credentials from this mount. + + cache + Only list credentials using this cache name (refer to get_creds for details). + + static + Only list static (``True``) or dynamic (``False``) credentials. + """ + creds_cache = vault.get_lease_store(__opts__, __context__) + info = creds_cache.list_info( + match=vaultdb.create_cache_pattern(name=name, mount=mount, cache=cache, static=static) + ) + for lease in info.values(): + for val in ("creation_time", "expire_time"): + if val in lease: + if val == "expire_time": + # The Vault util module stores the timestamp in the local time zone + # FIXME? + lease["expires_in"] = int(lease[val] - datetime.now().timestamp()) + lease["expired"] = not lease["expires_in"] > 0 + lease[val] = datetime.fromtimestamp(lease[val], tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M:%S" + ) + return info + + +def renew_cached(name=None, mount=None, cache=None, static=None, increment=None): + """ + Renew cached database credentials matching specified parameters. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.renew_cached name=myrole mount=database + salt '*' vault_db.renew_cached mount=database + salt '*' vault_db.renew_cached + + name + Only renew credentials using this role name. + + mount + Only renew credentials from this mount. + + cache + Only renew credentials using this cache name (refer to get_creds for details). + + static + Only renew static (``True``) or dynamic (``False``) credentials. + + increment + Request the leases to be valid for this amount of time from the current + point of time onwards. Can also be used to reduce the validity period. + The server might not honor this increment. + Can be an integer (seconds) or a time string like ``1h``. Optional. + If unset, defaults to what was set on the lease during creation or + the lease's default TTL. + """ + creds_cache = vault.get_lease_store(__opts__, __context__) + return creds_cache.renew_cached( + match=vaultdb.create_cache_pattern(name=name, mount=mount, cache=cache, static=static), + increment=increment, + ) + + +def rotate_static_role(name, mount="database"): + """ + Rotate Static Role credentials stored for a given role name. + + `API method docs static `_. + + CLI Example: + + .. code-block:: bash + + salt '*' vault_db.rotate_static_role mystaticrole + + name + The name of the database role. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + endpoint = f"{mount}/rotate-role/{name}" + try: + return vault.query("POST", endpoint, __opts__, __context__) + except vault.VaultException as err: + raise CommandExecutionError(f"{err.__class__}: {err}") from err diff --git a/src/saltext/vault/states/vault_db.py b/src/saltext/vault/states/vault_db.py new file mode 100644 index 00000000..efeed422 --- /dev/null +++ b/src/saltext/vault/states/vault_db.py @@ -0,0 +1,765 @@ +""" +Manage the Vault database secret engine, request and cache +leased database credentials. + +.. important:: + This module requires the general :ref:`Vault setup `. +""" +import logging + +import saltext.vault.utils.vault as vault +import saltext.vault.utils.vault.db as vaultdb +from salt.exceptions import CommandExecutionError +from salt.exceptions import SaltInvocationError + +log = logging.getLogger(__name__) + + +def connection_present( + name, + plugin, + version=None, + verify=True, + allowed_roles=None, + root_rotation_statements=None, + password_policy=None, + rotate=True, + force=False, + mount="database", + **kwargs, +): + """ + Ensure a database connection is present as specified. + + name + The name of the database connection. + + plugin + The name of the database plugin. Known plugins to this module are: + ``cassandra``, ``couchbase``, ``elasticsearch``, ``influxdb``, ``hanadb``, ``mongodb``, + ``mongodb_atlas``, ``mssql``, ``mysql``, ``oracle``, ``postgresql``, ``redis``, + ``redis_elasticache``, ``redshift``, ``snowflake``. + If you pass an unknown plugin, make sure its Vault-internal name can be formatted + as ``{plugin}-database-plugin`` and to pass all required parameters as kwargs. + + version + Specifies the semantic version of the plugin to use for this connection. + + verify + Verify the connection during initial configuration. Defaults to True. + + allowed_roles + List of the roles allowed to use this connection. ``["*"]`` means any role + can use this connection. Defaults to empty (no role can use it). + + root_rotation_statements + Specifies the database statements to be executed to rotate the root user's credentials. + See the plugin's API page for more information on support and formatting for this parameter. + + password_policy + The name of the password policy to use when generating passwords for this database. + If not specified, this will use a default policy defined as: + 20 characters with at least 1 uppercase, 1 lowercase, 1 number, and 1 dash character. + + rotate + Rotate the root credentials after plugin setup. Defaults to True. + + force + When the plugin changes, this state fails to protect from accidental errors. + Set force to True to delete existing connections with the same name and a + different plugin type. Defaults to False. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + + kwargs + Different plugins require different parameters. You need to make sure that you pass them + as supplemental keyword arguments. For known plugins, the required arguments will + be checked. + """ + ret = {"name": name, "result": True, "comment": "", "changes": {}} + kwargs = {k: v for k, v in kwargs.items() if not k.startswith("_")} + + def _diff_params(current): + nonlocal version, allowed_roles, root_rotation_statements, password_policy, kwargs + diff_params = ( + ("plugin_version", version), + ("allowed_roles", allowed_roles), + ("root_credentials_rotate_statements", root_rotation_statements), + ("password_policy", password_policy), + ) + changed = {} + for param, arg in diff_params: + if arg is None: + continue + if param not in current or current[param] != arg: + changed.update({param: {"old": current.get(param), "new": arg}}) + for param, val in kwargs.items(): + if param == "password": + # password is not reported + continue + if ( + param not in current["connection_details"] + or current["connection_details"][param] != val + ): + changed.update( + {param: {"old": current["connection_details"].get(param), "new": val}} + ) + return changed + + try: + current = __salt__["vault_db.fetch_connection"](name, mount=mount) + changes = {} + + if current: + if current["plugin_name"] != vaultdb.get_plugin_name(plugin): + if not force: + raise CommandExecutionError( + "Cannot change plugin type without deleting the existing connection. " + "Set force: true to override." + ) + if not __opts__["test"]: + __salt__["vault_db.delete_connection"](name, mount=mount) + ret["changes"]["deleted for plugin change"] = name + current = None + else: + changes = _diff_params(current) + if not changes: + ret["comment"] = "Connection is present as specified" + return ret + + if __opts__["test"]: + ret["result"] = None + ret[ + "comment" + ] = f"Connection `{name}` would have been {'updated' if current else 'created'}" + ret["changes"].update(changes) + if not current: + ret["changes"]["created"] = name + return ret + + if current and "password" in kwargs: + kwargs.pop("password") + + __salt__["vault_db.write_connection"]( + name, + plugin, + version=version, + verify=verify, + allowed_roles=allowed_roles, + root_rotation_statements=root_rotation_statements, + password_policy=password_policy, + rotate=rotate, + mount=mount, + **kwargs, + ) + new = __salt__["vault_db.fetch_connection"](name, mount=mount) + + if new is None: + raise CommandExecutionError( + "There were no errors during role management, but it is reported as absent." + ) + if not current: + ret["changes"]["created"] = name + + new_diff = _diff_params(new) + if new_diff: + ret["result"] = False + ret["comment"] = ( + "There were no errors during connection management, but " + f"the reported parameters do not match: {new_diff}" + ) + return ret + ret["changes"].update(changes) + + except CommandExecutionError as err: + ret["result"] = False + ret["comment"] = str(err) + # do not reset changes + + return ret + + +def connection_absent(name, mount="database"): + """ + Ensure a database connection is absent. + + name + The name of the connection. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + ret = {"name": name, "result": True, "comment": "", "changes": {}} + + try: + current = __salt__["vault_db.fetch_connection"](name, mount=mount) + + if current is None: + ret["comment"] = f"Connection `{name}` is already absent." + return ret + + ret["changes"]["deleted"] = name + + if __opts__["test"]: + ret["result"] = None + ret["comment"] = f"Connection `{name}` would have been deleted." + return ret + + __salt__["vault_db.delete_connection"](name, mount=mount) + + if __salt__["vault_db.fetch_connection"](name, mount=mount) is not None: + raise CommandExecutionError( + "There were no errors during connection deletion, " + "but it is still reported as present." + ) + ret["comment"] = f"Connection `{name}` has been deleted." + + except CommandExecutionError as err: + ret["result"] = False + ret["comment"] = str(err) + ret["changes"] = {} + + return ret + + +def role_present( + name, + connection, + creation_statements, + default_ttl=None, + max_ttl=None, + revocation_statements=None, + rollback_statements=None, + renew_statements=None, + credential_type=None, + credential_config=None, + mount="database", +): + """ + Ensure a regular database role is present as specified. + + name + The name of the database role. + + connection + The name of the database connection this role applies to. + + creation_statements + Specifies a list of database statements executed to create and configure a user, + usually templated with {{name}} and {{password}}. Required. + + default_ttl + Specifies the TTL for the leases associated with this role. Accepts time suffixed + strings (1h) or an integer number of seconds. Defaults to system/engine default TTL time. + + max_ttl + Specifies the maximum TTL for the leases associated with this role. Accepts time suffixed + strings (1h) or an integer number of seconds. Defaults to sys/mounts's default TTL time; + this value is allowed to be less than the mount max TTL (or, if not set, + the system max TTL), but it is not allowed to be longer. + + revocation_statements + Specifies a list of database statements to be executed to revoke a user. + + rollback_statements + Specifies a list of database statements to be executed to rollback a create operation + in the event of an error. Availability and formatting depend on the specific plugin. + + renew_statements + Specifies a list of database statements to be executed to renew a user. + Availability and formatting depend on the specific plugin. + + credential_type + Specifies the type of credential that will be generated for the role. + Options include: ``password``, ``rsa_private_key``. Defaults to ``password``. + See the plugin's API page for credential types supported by individual databases. + + credential_config + Specifies the configuration for the given ``credential_type`` as a mapping. + For ``password``, only ``password_policy`` can be passed. + For ``rsa_private_key``, ``key_bits`` (defaults to 2048) and ``format`` + (defaults to ``pkcs8``) are available. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + ret = {"name": name, "result": True, "comment": "", "changes": {}} + + if not isinstance(creation_statements, list): + creation_statements = [creation_statements] + if revocation_statements and not isinstance(revocation_statements, list): + revocation_statements = [revocation_statements] + if rollback_statements and not isinstance(rollback_statements, list): + rollback_statements = [rollback_statements] + if renew_statements and not isinstance(renew_statements, list): + renew_statements = [renew_statements] + + def _diff_params(current): + nonlocal connection, creation_statements, default_ttl, max_ttl, revocation_statements + nonlocal rollback_statements, renew_statements, credential_type, credential_config + + diff_params = ( + ("db_name", connection), + ("creation_statements", creation_statements), + ("default_ttl", vault.timestring_map(default_ttl)), + ("max_ttl", vault.timestring_map(max_ttl)), + ("revocation_statements", revocation_statements), + ("rollback_statements", rollback_statements), + ("renew_statements", renew_statements), + ("credential_type", credential_type), + ("credential_config", credential_config), + ) + changed = {} + for param, arg in diff_params: + if arg is None: + continue + # Strip statements to avoid tripping over final newlines + if param.endswith("statements"): + arg = [x.rstrip() for x in arg] + if param in current: + current[param] = [x.rstrip() for x in current[param]] + if param not in current or current[param] != arg: + changed.update({param: {"old": current.get(param), "new": arg}}) + return changed + + try: + current = __salt__["vault_db.fetch_role"](name, static=False, mount=mount) + + if current: + changed = _diff_params(current) + if not changed: + ret["comment"] = "Role is present as specified" + return ret + ret["changes"].update(changed) + + if __opts__["test"]: + ret["result"] = None + ret["comment"] = f"Role `{name}` would have been {'updated' if current else 'created'}" + if not current: + ret["changes"]["created"] = name + return ret + + __salt__["vault_db.write_role"]( + name, + connection, + creation_statements, + default_ttl=default_ttl, + max_ttl=max_ttl, + revocation_statements=revocation_statements, + rollback_statements=rollback_statements, + renew_statements=renew_statements, + credential_type=credential_type, + credential_config=credential_config, + mount=mount, + ) + new = __salt__["vault_db.fetch_role"](name, static=False, mount=mount) + + if new is None: + raise CommandExecutionError( + "There were no errors during role management, but it is reported as absent." + ) + + if not current: + ret["changes"]["created"] = name + + new_diff = _diff_params(new) + if new_diff: + ret["result"] = False + ret["comment"] = ( + "There were no errors during role management, but " + f"the reported parameters do not match: {new_diff}" + ) + return ret + + except (CommandExecutionError, SaltInvocationError) as err: + ret["result"] = False + ret["comment"] = str(err) + ret["changes"] = {} + + return ret + + +def role_absent(name, static=False, mount="database"): + """ + Ensure a database role is absent. + + name + The name of the role. + + static + Whether this role is static. Defaults to False. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + ret = {"name": name, "result": True, "comment": "", "changes": {}} + + try: + current = __salt__["vault_db.fetch_role"](name, static=static, mount=mount) + + if current is None: + ret["comment"] = f"Role `{name}` is already absent." + return ret + + ret["changes"]["deleted"] = name + + if __opts__["test"]: + ret["result"] = None + ret["comment"] = f"Role `{name}` would have been deleted." + return ret + + __salt__["vault_db.delete_role"](name, static=static, mount=mount) + + if __salt__["vault_db.fetch_role"](name, static=static, mount=mount) is not None: + raise CommandExecutionError( + "There were no errors during role deletion, but it is still reported as present." + ) + ret["comment"] = f"Role `{name}` has been deleted." + + except CommandExecutionError as err: + ret["result"] = False + ret["comment"] = str(err) + ret["changes"] = {} + + return ret + + +def static_role_present( + name, + connection, + username, + rotation_period, + rotation_statements=None, + credential_type=None, + credential_config=None, + mount="database", +): + """ + Ensure a database Static Role is present as specified. + + name + The name of the database role. + + connection + The name of the database connection this role applies to. + + username + The username to manage. + + rotation_period + Specifies the amount of time Vault should wait before rotating the password. + The minimum is ``5s``. + + rotation_statements + Specifies the database statements to be executed to rotate the password for the + configured database user. Not every plugin type will support this functionality. + + credential_type + Specifies the type of credential that will be generated for the role. + Options include: ``password``, ``rsa_private_key``. Defaults to ``password``. + See the plugin's API page for credential types supported by individual databases. + + credential_config + Specifies the configuration for the given ``credential_type`` as a mapping. + For ``password``, only ``password_policy`` can be passed. + For ``rsa_private_key``, ``key_bits`` (defaults to 2048) and ``format`` + (defaults to ``pkcs8``) are available. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + ret = {"name": name, "result": True, "comment": "", "changes": {}} + + if rotation_statements and not isinstance(rotation_statements, list): + rotation_statements = [rotation_statements] + + def _diff_params(current): + nonlocal connection, username, rotation_period, rotation_statements, credential_type, credential_config + diff_params = ( + ("db_name", connection), + ("username", username), + ("rotation_period", vault.timestring_map(rotation_period)), + ("rotation_statements", rotation_statements), + ("credential_type", credential_type), + ("credential_config", credential_config), + ) + changed = {} + for param, arg in diff_params: + if arg is None: + continue + if param not in current or current[param] != arg: + changed.update({param: {"old": current.get(param), "new": arg}}) + return changed + + try: + current = __salt__["vault_db.fetch_role"](name, static=True, mount=mount) + + if current: + changed = _diff_params(current) + if not changed: + ret["comment"] = "Role is present as specified" + return ret + ret["changes"].update(changed) + + if __opts__["test"]: + ret["result"] = None + ret["comment"] = f"Role `{name}` would have been {'updated' if current else 'created'}" + if not current: + ret["changes"]["created"] = name + return ret + + __salt__["vault_db.write_static_role"]( + name, + connection, + username, + rotation_period, + rotation_statements=None, + credential_type=credential_type, + credential_config=credential_config, + mount=mount, + ) + new = __salt__["vault_db.fetch_role"](name, static=True, mount=mount) + + if new is None: + raise CommandExecutionError( + "There were no errors during role management, but it is reported as absent." + ) + + if not current: + ret["changes"]["created"] = name + + new_diff = _diff_params(new) + if new_diff: + ret["result"] = False + ret["comment"] = ( + "There were no errors during role management, but " + f"the reported parameters do not match: {new_diff}" + ) + return ret + + except (CommandExecutionError, SaltInvocationError) as err: + ret["result"] = False + ret["comment"] = str(err) + ret["changes"] = {} + + return ret + + +def creds_cached( + name, + static=False, + cache=None, + valid_for=None, + renew_increment=None, + revoke_delay=None, + meta=None, + mount="database", + **kwargs, # pylint: disable=unused-argument +): + """ + Ensure valid credentials are present in the minion's cache based on the named role. + Supports ``mod_beacon``. + + .. note:: + + This function is mosly intended to associate a specific credential with + a beacon that warns about expiry and allows to run an associated state to + reconfigure an application with new credentials. + + name + The name of the database role. + + static + Whether this role is static. Defaults to False. + + cache + A variable cache suffix to be able to use multiple distinct credentials + using the same role on the same minion. + Ignored when ``static`` is true. + + .. note:: + + This uses the same cache backend as the Vault integration, so make + sure you configure a persistent backend like ``disk`` if you expect + the credentials to survive a single run. + + valid_for + Ensure the credentials are valid for at least this amount of time, + otherwise request new ones. + This can be an integer, which will be interpreted as seconds, or a time string + using the same format as Vault does: + Suffix ``s`` for seconds, ``m`` for minuts, ``h`` for hours, ``d`` for days. + Defaults to ``0``. + + renew_increment + When using cache and ``valid_for`` results in a renewal attempt, request this + amount of time extension on the lease. This will be cached together with the + lease and might be used by other modules later. + + revoke_delay + When using cache and ``valid_for`` results in a revocation, set the lease + validity to this value to allow a short amount of delay between the issuance + of the new lease and the revocation of the old one. Defaults to ``60``. + This will be cached together with the lease and might be used by other + modules later. + + meta + When using cache, this value will be cached together with the lease. It will + be emitted by the ``vault_lease`` beacon module whenever a lease is + running out (usually because it cannot be extended further). It is intended + to support the reactor in deciding what needs to be done in order + to to reconfigure dependent, Vault-unaware software with newly issued + credentials. Entirely optional. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + ret = { + "name": name, + "result": True, + "comment": "The credentials are already cached and valid", + "changes": {}, + } + + cached = __salt__["vault_db.list_cached"](name, static=static, cache=cache, mount=mount) + verb = "issue" + if cached: + info = cached[next(iter(cached))] + for attr, val in ( + ("min_ttl", valid_for), + ("renew_increment", renew_increment), + ("revoke_delay", revoke_delay), + ("meta", meta), + ): + if val is not None and info.get(attr) != val: + # Not sure if meta-changes should actually be reported here + ret["changes"][attr] = {"old": info.get(attr), "new": val} + verb = "edite" + if info["expires_in"] <= vault.timestring_map(valid_for): + ret["changes"]["expiry"] = True + verb = "reissue" + if not ret["changes"]: + return ret + else: + ret["changes"]["new"] = True + if __opts__["test"]: + ret["result"] = None + ret["comment"] = f"The credentials would have been {verb}d" + return ret + __salt__["vault_db.get_creds"]( + name, + static=static, + cache=cache or True, + valid_for=valid_for, + renew_increment=renew_increment, + revoke_delay=revoke_delay, + meta=meta, + mount=mount, + _warn_about_attr_change=False, + ) + new_cached = __salt__["vault_db.list_cached"](name, static=static, cache=cache, mount=mount) + if not new_cached: + raise CommandExecutionError( + "Could not find cached credentials after issuing, this is likely a bug" + ) + if verb == "reissue": + # Ensure the reporting is correct, usually the lease would + # just be renewed. + if new_cached[next(iter(cached))]["lease_id"] == info["lease_id"]: + verb = "renewe" + + ret["comment"] = f"The credentials have been {verb}d" + return ret + + +def creds_uncached( + name, static=False, cache=None, mount="database", **kwargs +): # pylint: disable=unused-argument + """ + Ensure credentials are absent in the minion's cache based on the named role. + Supports ``mod_beacon``. + + .. note:: + + This function is mosly intended to associate a specific credential with + a beacon that warns about expiry and allows to run an associated state to + reconfigure an application with new credentials. + + name + The name of the database role. + + static + Whether this role is static. Defaults to False. + + cache + A variable cache suffix to be able to use multiple distinct credentials + using the same role on the same minion. + Ignored when ``static`` is true. + + mount + The mount path the database backend is mounted to. Defaults to ``database``. + """ + ret = { + "name": name, + "result": True, + "comment": "No matching credentials present", + "changes": {}, + } + + cached = __salt__["vault_db.list_cached"](name, static=static, cache=cache, mount=mount) + if not cached: + return ret + ret["changes"]["revoked"] = True + if __opts__["test"]: + ret["result"] = None + ret["comment"] = "The credentials would have been revoked" + return ret + __salt__["vault_db.clear_cached"](name, static=static, cache=cache or True, mount=mount) + ret["comment"] = "The credentials have been revoked" + return ret + + +def mod_beacon(name, sfun=None, static=False, cache=None, mount="database", **kwargs): + """ + Associates a Vault lease with a ``vault_lease`` beacon and + possibly a state. + + beacon_interval + The interval to run the beacon in. Defaults to 60. + + min_ttl + If this minimum TTL on the lease is undercut, the beacon will + fire an event. Defaults to 0. + """ + ret = {"name": name, "changes": {}, "result": True, "comment": ""} + supported_funcs = ["creds_cached", "creds_uncached"] + + if sfun not in supported_funcs: + ret["result"] = False + ret["comment"] = f"'vault_db.{sfun}' does not work with mod_beacon" + return ret + if not kwargs.get("beacon"): + ret["comment"] = "Not managing beacon" + return ret + lease = vaultdb.create_cache_pattern(name, mount=mount, static=static, cache=cache) + beacon_module = "vault_lease" + beacon_name = f"{beacon_module}_{lease}" + if sfun == "creds_uncached": + beacon_kwargs = { + "name": beacon_name, + "beacon_module": beacon_module, + } + bfun = "absent" + elif sfun == "creds_cached": + beacon_kwargs = { + "name": beacon_name, + "beacon_module": beacon_module, + "interval": kwargs.get("beacon_interval", 60), + "lease": lease, + "min_ttl": kwargs.get("min_ttl", 0), + "meta": kwargs.get("meta"), + "check_server": kwargs.get("check_server", False), + } + bfun = "present" + return __states__[f"beacon.{bfun}"](**beacon_kwargs) diff --git a/src/saltext/vault/utils/vault/db.py b/src/saltext/vault/utils/vault/db.py new file mode 100644 index 00000000..623f53f8 --- /dev/null +++ b/src/saltext/vault/utils/vault/db.py @@ -0,0 +1,158 @@ +from salt.utils.immutabletypes import freeze + + +PLUGINS = freeze( + { + "cassandra": { + "name": "cassandra", + "required": [ + "hosts", + "username", + "password", + ], + }, + "couchbase": { + "name": "couchbase", + "required": [ + "hosts", + "username", + "password", + ], + }, + "elasticsearch": { + "name": "elasticsearch", + "required": [ + "url", + "username", + "password", + ], + }, + "influxdb": { + "name": "influxdb", + "required": [ + "host", + "username", + "password", + ], + }, + "hanadb": { + "name": "hana", + "required": [ + "connection_url", + ], + }, + "mongodb": { + "name": "mongodb", + "required": [ + "connection_url", + ], + }, + "mongodb_atlas": { + "name": "mongodbatlas", + "required": [ + "public_key", + "private_key", + "project_id", + ], + }, + "mssql": { + "name": "mssql", + "required": [ + "connection_url", + ], + }, + "mysql": { + "name": "mysql", + "required": [ + "connection_url", + ], + }, + "oracle": { + "name": "oracle", + "required": [ + "connection_url", + ], + }, + "postgresql": { + "name": "postgresql", + "required": [ + "connection_url", + ], + }, + "redis": { + "name": "redis", + "required": [ + "host", + "port", + "username", + "password", + ], + }, + "redis_elasticache": { + "name": "redis-elasticache", + "required": [ + "url", + "username", + "password", + ], + }, + "redshift": { + "name": "redshift", + "required": [ + "connection_url", + ], + }, + "snowflake": { + "name": "snowflake", + "required": [ + "connection_url", + ], + }, + "default": { + "name": "", + "required": [], + }, + } +) + + +def get_plugin_meta(name): + """ + Get meta information for a plugin with this name, + excluding the `-database-plugin` suffix. + """ + return PLUGINS.get(name, PLUGINS["default"]) + + +def get_plugin_name(name): + """ + Get the name of a plugin as rendered by this module. This is a utility for the state + module primarily. + """ + plugin_name = PLUGINS.get(name, {"name": name})["name"] + return f"{plugin_name}-database-plugin" + + +def create_cache_pattern(name=None, mount=None, cache=None, static=None): + """ + Render a match pattern for operating on cached leases. + Unset parameters will result in a ``*`` glob. + + name + The name of the database role. + + static + Whether the role is static. + + cache + Filter by cache name (refer to get_creds for details). + + mount + The mount path the associated database backend is mounted to. + """ + ptrn = ["db"] + ptrn.append("*" if mount is None else mount) + ptrn.append("*" if static is None else "static" if static else "dynamic") + ptrn.append("*" if name is None else name) + ptrn.append("*" if cache is None else "default" if cache is True else cache) + return ".".join(ptrn) diff --git a/tests/functional/modules/test_vault_db.py b/tests/functional/modules/test_vault_db.py new file mode 100644 index 00000000..51c6df29 --- /dev/null +++ b/tests/functional/modules/test_vault_db.py @@ -0,0 +1,369 @@ +import time + +import pytest +from saltfactories.utils import random_string + +from tests.support.mysql import create_mysql_combo # pylint: disable=unused-import +from tests.support.mysql import mysql_combo # pylint: disable=unused-import +from tests.support.mysql import mysql_container # pylint: disable=unused-import +from tests.support.mysql import MySQLImage +from tests.support.vault import vault_delete +from tests.support.vault import vault_disable_secret_engine +from tests.support.vault import vault_enable_secret_engine +from tests.support.vault import vault_list +from tests.support.vault import vault_revoke +from tests.support.vault import vault_write + + +pytestmark = [ + pytest.mark.slow_test, + pytest.mark.skip_if_binaries_missing("dockerd", "vault", "getent"), + pytest.mark.usefixtures("vault_container_version"), + pytest.mark.parametrize("vault_container_version", ["latest"], indirect=True), +] + + +@pytest.fixture(scope="module") +def minion_config_overrides(vault_port): + return { + "vault": { + "auth": { + "method": "token", + "token": "testsecret", + }, + "cache": { + "backend": "disk", # ensure a persistent cache is available for get_creds + }, + "server": { + "url": f"http://127.0.0.1:{vault_port}", + }, + } + } + + +@pytest.fixture(scope="module") +def mysql_image(): + version = "10.3" + return MySQLImage( + name="mariadb", + tag=version, + container_id=random_string(f"mariadb-{version}-"), + ) + + +@pytest.fixture +def role_args_common(): + return { + "db_name": "testdb", + "creation_statements": r"CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';GRANT SELECT ON *.* TO '{{name}}'@'%';", + } + + +@pytest.fixture +def testrole(): + return { + "default_ttl": 3600, + "max_ttl": 86400, + } + + +@pytest.fixture +def testreissuerole(): + return { + "default_ttl": 180, + "max_ttl": 180, + } + + +@pytest.fixture +def teststaticrole(mysql_container): + return { + "db_name": "testdb", + "rotation_period": 86400, + "username": mysql_container.mysql_user, + } + + +@pytest.fixture +def testdb(mysql_container): + # This uses the default IP address of the host on the default network + # (hardcoded) because I could not get hostname resolution working properly. + return { + "plugin_name": "mysql-database-plugin", + "connection_url": f"{{{{username}}}}:{{{{password}}}}@tcp(172.17.0.1:{mysql_container.mysql_port})/", + "allowed_roles": "testrole,teststaticrole,testreissuerole", + "username": "root", + "password": mysql_container.mysql_passwd, + } + + +@pytest.fixture(scope="module", autouse=True) +def db_engine(vault_container_version): # pylint: disable=unused-argument + assert vault_enable_secret_engine("database") + yield + assert vault_disable_secret_engine("database") + + +@pytest.fixture +def connection_setup(testdb): + try: + vault_write("database/config/testdb", **testdb) + assert "testdb" in vault_list("database/config") + yield + finally: + # prevent dangling leases, which prevent disabling the secret engine + assert vault_revoke("database/creds", prefix=True) + if "testdb" in vault_list("database/config"): + vault_delete("database/config/testdb") + assert "testdb" not in vault_list("database/config") + + +@pytest.fixture(params=[["testrole"]]) +def roles_setup(connection_setup, request, role_args_common): # pylint: disable=unused-argument + try: + for role_name in request.param: + role_args = request.getfixturevalue(role_name) + role_args.update(role_args_common) + vault_write(f"database/roles/{role_name}", **role_args) + assert role_name in vault_list("database/roles") + yield + finally: + for role_name in request.param: + if role_name in vault_list("database/roles"): + vault_delete(f"database/roles/{role_name}") + assert role_name not in vault_list("database/roles") + + +@pytest.fixture +def role_static_setup(connection_setup, teststaticrole): # pylint: disable=unused-argument + role_name = "teststaticrole" + try: + vault_write(f"database/static-roles/{role_name}", **teststaticrole) + assert role_name in vault_list("database/static-roles") + yield + finally: + if role_name in vault_list("database/static-roles"): + vault_delete(f"database/static-roles/{role_name}") + assert role_name not in vault_list("database/static-roles") + + +@pytest.fixture +def vault_db(modules): + try: + yield modules.vault_db + finally: + # prevent dangling leases, which prevent disabling the secret engine + assert vault_revoke("database/creds", prefix=True) + if "testdb" in vault_list("database/config"): + vault_delete("database/config/testdb") + assert "testdb" not in vault_list("database/config") + if "testrole" in vault_list("database/roles"): + vault_delete("database/roles/testrole") + assert "testrole" not in vault_list("database/roles") + if "teststaticrole" in vault_list("database/static-roles"): + vault_delete("database/static-roles/teststaticrole") + assert "teststaticrole" not in vault_list("database/static-roles") + + +@pytest.mark.usefixtures("connection_setup") +def test_list_connections(vault_db): + ret = vault_db.list_connections() + assert ret == ["testdb"] + + +@pytest.mark.usefixtures("connection_setup") +def test_fetch_connection(vault_db, testdb): + ret = vault_db.fetch_connection("testdb") + assert ret + for var, val in testdb.items(): + if var == "password": + continue + if var in ["connection_url", "username"]: + assert var in ret["connection_details"] + assert ret["connection_details"][var] == val + else: + assert var in ret + if var == "allowed_roles": + assert ret[var] == list(val.split(",")) + else: + assert ret[var] == val + + +@pytest.mark.usefixtures("testdb") +def test_write_connection(vault_db, mysql_container): + args = { + "plugin": "mysql", + "connection_url": f"{{{{username}}}}:{{{{password}}}}@tcp(172.17.0.1:{mysql_container.mysql_port})/", + "allowed_roles": ["testrole", "teststaticrole"], + "username": "root", + "password": mysql_container.mysql_passwd, + "rotate": False, + } + ret = vault_db.write_connection("testdb", **args) + assert ret + assert "testdb" in vault_list("database/config") + + +@pytest.mark.usefixtures("connection_setup") +def test_delete_connection(vault_db): + ret = vault_db.delete_connection("testdb") + assert ret + assert "testdb" not in vault_list("database/config") + + +@pytest.mark.usefixtures("connection_setup") +def test_reset_connection(vault_db): + ret = vault_db.reset_connection("testdb") + assert ret + + +@pytest.mark.usefixtures("roles_setup") +def test_list_roles(vault_db): + ret = vault_db.list_roles() + assert ret == ["testrole"] + + +@pytest.mark.usefixtures("role_static_setup") +def test_list_roles_static(vault_db): + ret = vault_db.list_roles(static=True) + assert ret == ["teststaticrole"] + + +@pytest.mark.usefixtures("roles_setup") +def test_fetch_role(vault_db, testrole): + ret = vault_db.fetch_role("testrole") + assert ret + for var, val in testrole.items(): + assert var in ret + if var == "creation_statements": + assert ret[var] == [val] + else: + assert ret[var] == val + + +@pytest.mark.usefixtures("role_static_setup") +def test_fetch_role_static(vault_db, teststaticrole): + ret = vault_db.fetch_role("teststaticrole", static=True) + assert ret + for var, val in teststaticrole.items(): + assert var in ret + assert ret[var] == val + + +@pytest.mark.usefixtures("connection_setup") +def test_write_role(vault_db): + args = { + "connection": "testdb", + "creation_statements": r"CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';GRANT SELECT ON *.* TO '{{name}}'@'%';", + } + ret = vault_db.write_role("testrole", **args) + assert ret + assert "testrole" in vault_list("database/roles") + + +@pytest.mark.usefixtures("connection_setup") +def test_write_static_role(vault_db, mysql_container): + args = { + "connection": "testdb", + "username": mysql_container.mysql_user, + "rotation_period": 86400, + } + ret = vault_db.write_static_role("teststaticrole", **args) + assert ret + assert "teststaticrole" in vault_list("database/static-roles") + + +@pytest.mark.usefixtures("roles_setup") +def test_delete_role(vault_db): + ret = vault_db.delete_role("testrole") + assert ret + assert "testrole" not in vault_list("database/roles") + + +@pytest.mark.usefixtures("role_static_setup") +def test_delete_role_static(vault_db): + ret = vault_db.delete_role("teststaticrole", static=True) + assert ret + assert "teststaticrole" not in vault_list("database/static-roles") + + +@pytest.mark.usefixtures("roles_setup") +def test_get_creds(vault_db): + ret = vault_db.get_creds("testrole", cache=False) + assert ret + assert "username" in ret + assert "password" in ret + + +@pytest.mark.usefixtures("role_static_setup") +def test_get_creds_static(vault_db, teststaticrole): + ret = vault_db.get_creds("teststaticrole", static=True, cache=False) + assert ret + assert "username" in ret + assert "password" in ret + assert ret["username"] == teststaticrole["username"] + + +@pytest.mark.usefixtures("roles_setup") +def test_get_creds_cached(vault_db): + ret = vault_db.get_creds("testrole", cache=True) + assert ret + assert "username" in ret + assert "password" in ret + ret_new = vault_db.get_creds("testrole", cache=True) + assert ret_new + assert "username" in ret_new + assert "password" in ret_new + assert ret_new["username"] == ret["username"] + assert ret_new["password"] == ret["password"] + + +@pytest.mark.usefixtures("roles_setup") +def test_get_creds_cached_multiple(vault_db): + ret = vault_db.get_creds("testrole", cache="one") + assert ret + assert "username" in ret + assert "password" in ret + ret_new = vault_db.get_creds("testrole", cache="two") + assert ret_new + assert "username" in ret_new + assert "password" in ret_new + assert ret_new["username"] != ret["username"] + assert ret_new["password"] != ret["password"] + assert vault_db.get_creds("testrole", cache="one") == ret + assert vault_db.get_creds("testrole", cache="two") == ret_new + + +@pytest.mark.usefixtures("roles_setup") +@pytest.mark.parametrize("roles_setup", [["testreissuerole"]], indirect=True) +def test_get_creds_cached_valid_for_reissue(vault_db, testreissuerole): + """ + Test that valid cached credentials that do not fulfill valid_for + and cannot be renewed as requested are reissued + """ + ret = vault_db.get_creds("testreissuerole", cache=True) + assert ret + assert "username" in ret + assert "password" in ret + # 3 seconds because of leeway in lease validity check after renewals + time.sleep(3) + ret_new = vault_db.get_creds( + "testreissuerole", cache=True, valid_for=testreissuerole["default_ttl"] + ) + assert ret_new + assert "username" in ret_new + assert "password" in ret_new + assert ret_new["username"] != ret["username"] + assert ret_new["password"] != ret["password"] + + +@pytest.mark.usefixtures("role_static_setup") +def test_rotate_static_role(vault_db): + ret = vault_db.get_creds("teststaticrole", static=True, cache=False) + assert ret + old_pw = ret["password"] + ret = vault_db.rotate_static_role("teststaticrole") + assert ret + ret = vault_db.get_creds("teststaticrole", static=True, cache=False) + assert ret + assert ret["password"] != old_pw diff --git a/tests/functional/states/test_vault_db.py b/tests/functional/states/test_vault_db.py new file mode 100644 index 00000000..7d52f32c --- /dev/null +++ b/tests/functional/states/test_vault_db.py @@ -0,0 +1,375 @@ +import pytest +from saltfactories.utils import random_string + +from tests.support.mysql import create_mysql_combo # pylint: disable=unused-import +from tests.support.mysql import mysql_combo # pylint: disable=unused-import +from tests.support.mysql import mysql_container # pylint: disable=unused-import +from tests.support.mysql import MySQLImage +from tests.support.vault import vault_delete +from tests.support.vault import vault_disable_secret_engine +from tests.support.vault import vault_enable_secret_engine +from tests.support.vault import vault_list +from tests.support.vault import vault_read +from tests.support.vault import vault_revoke +from tests.support.vault import vault_write + +pytestmark = [ + pytest.mark.slow_test, + pytest.mark.skip_if_binaries_missing("dockerd", "vault", "getent"), + pytest.mark.usefixtures("vault_container_version"), + pytest.mark.parametrize("vault_container_version", ["latest"], indirect=True), +] + + +@pytest.fixture(scope="module") +def minion_config_overrides(vault_port): + return { + "vault": { + "auth": { + "method": "token", + "token": "testsecret", + }, + "server": { + "url": f"http://127.0.0.1:{vault_port}", + }, + } + } + + +@pytest.fixture(scope="module") +def mysql_image(): + version = "10.5" + return MySQLImage( + name="mariadb", + tag=version, + container_id=random_string(f"mariadb-{version}-"), + ) + + +@pytest.fixture +def role_args_common(): + return { + "db_name": "testdb", + "creation_statements": r"CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';GRANT SELECT ON *.* TO '{{name}}'@'%';", + } + + +@pytest.fixture +def testrole(): + return { + "default_ttl": 3600, + "max_ttl": 86400, + } + + +@pytest.fixture +def teststaticrole(mysql_container): + return { + "db_name": "testdb", + "rotation_period": 86400, + "username": mysql_container.mysql_user, + } + + +@pytest.fixture +def testdb(mysql_container): + # This uses the default IP address of the host on the default network + # (hardcoded) because I could not get hostname resolution working properly. + return { + "plugin_name": "mysql-database-plugin", + "connection_url": f"{{{{username}}}}:{{{{password}}}}@tcp(172.17.0.1:{mysql_container.mysql_port})/", + "allowed_roles": "testrole,teststaticrole", + "username": "root", + "password": mysql_container.mysql_passwd, + } + + +@pytest.fixture(scope="module", autouse=True) +def db_engine(): + assert vault_enable_secret_engine("database") + yield + assert vault_disable_secret_engine("database") + + +@pytest.fixture +def connection_setup(testdb): + try: + vault_write("database/config/testdb", **testdb) + assert "testdb" in vault_list("database/config") + yield + finally: + # prevent dangling leases, which prevent disabling the secret engine + assert vault_revoke("database/creds", prefix=True) + if "testdb" in vault_list("database/config"): + vault_delete("database/config/testdb") + assert "testdb" not in vault_list("database/config") + + +@pytest.fixture(params=[["testrole"]]) +def roles_setup(connection_setup, request, role_args_common): # pylint: disable=unused-argument + try: + for role_name in request.param: + role_args = request.getfixturevalue(role_name) + role_args.update(role_args_common) + vault_write(f"database/roles/{role_name}", **role_args) + assert role_name in vault_list("database/roles") + yield + finally: + for role_name in request.param: + if role_name in vault_list("database/roles"): + vault_delete(f"database/roles/{role_name}") + assert role_name not in vault_list("database/roles") + + +@pytest.fixture +def role_static_setup(connection_setup, teststaticrole): # pylint: disable=unused-argument + role_name = "teststaticrole" + try: + vault_write(f"database/static-roles/{role_name}", **teststaticrole) + assert role_name in vault_list("database/static-roles") + yield + finally: + if role_name in vault_list("database/static-roles"): + vault_delete(f"database/static-roles/{role_name}") + assert role_name not in vault_list("database/static-roles") + + +@pytest.fixture +def vault_db(states): + try: + yield states.vault_db + finally: + # prevent dangling leases, which prevent disabling the secret engine + assert vault_revoke("database/creds", prefix=True) + if "testdb" in vault_list("database/config"): + vault_delete("database/config/testdb") + assert "testdb" not in vault_list("database/config") + if "testrole" in vault_list("database/roles"): + vault_delete("database/roles/testrole") + assert "testrole" not in vault_list("database/roles") + if "teststaticrole" in vault_list("database/static-roles"): + vault_delete("database/static-roles/teststaticrole") + assert "teststaticrole" not in vault_list("database/static-roles") + + +@pytest.fixture +def connargs(mysql_container): + return { + "plugin": "mysql", + "connection_url": f"{{{{username}}}}:{{{{password}}}}@tcp(172.17.0.1:{mysql_container.mysql_port})/", + "allowed_roles": ["testrole", "teststaticrole"], + "username": "root", + "password": mysql_container.mysql_passwd, + "rotate": False, + } + + +@pytest.fixture +def roleargs(): + return { + "connection": "testdb", + "creation_statements": r"CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';GRANT SELECT ON *.* TO '{{name}}'@'%';", + } + + +@pytest.fixture +def roleargs_static(mysql_container): + return { + "connection": "testdb", + "username": mysql_container.mysql_user, + "rotation_period": 86400, + } + + +def test_connection_present(vault_db, connargs): + ret = vault_db.connection_present("testdb", **connargs) + assert ret.result + assert ret.changes + assert "created" in ret.changes + assert ret.changes["created"] == "testdb" + assert "testdb" in vault_list("database/config") + + +@pytest.mark.usefixtures("connection_setup") +def test_connection_present_no_changes(vault_db, connargs): + ret = vault_db.connection_present("testdb", **connargs) + assert ret.result + assert not ret.changes + + +@pytest.mark.usefixtures("connection_setup") +def test_connection_present_allowed_roles_change(vault_db, connargs): + connargs["allowed_roles"] = ["testrole", "teststaticrole", "newrole"] + ret = vault_db.connection_present("testdb", **connargs) + assert ret.result + assert ret.changes + assert "allowed_roles" in ret.changes + assert ( + vault_read("database/config/testdb")["data"]["allowed_roles"] == connargs["allowed_roles"] + ) + + +@pytest.mark.usefixtures("connection_setup") +def test_connection_present_new_param(vault_db, connargs): + connargs["username_template"] = r"{{random 20}}" + ret = vault_db.connection_present("testdb", **connargs) + assert ret.result + assert ret.changes + assert "username_template" in ret.changes + assert ( + vault_read("database/config/testdb")["data"]["connection_details"]["username_template"] + == connargs["username_template"] + ) + + +def test_connection_present_test_mode(vault_db, connargs): + ret = vault_db.connection_present("testdb", test=True, **connargs) + assert ret.result is None + assert ret.changes + assert "created" in ret.changes + assert ret.changes["created"] == "testdb" + assert "testdb" not in vault_list("database/config") + + +@pytest.mark.usefixtures("connection_setup") +def test_connection_absent(vault_db): + ret = vault_db.connection_absent("testdb") + assert ret.result + assert ret.changes + assert "deleted" in ret.changes + assert ret.changes["deleted"] == "testdb" + assert "testdb" not in vault_list("database/config") + + +def test_connection_absent_no_changes(vault_db): + ret = vault_db.connection_absent("testdb") + assert ret.result + assert not ret.changes + + +@pytest.mark.usefixtures("connection_setup") +def test_connection_absent_test_mode(vault_db): + ret = vault_db.connection_absent("testdb", test=True) + assert ret.result is None + assert ret.changes + assert "deleted" in ret.changes + assert ret.changes["deleted"] == "testdb" + assert "testdb" in vault_list("database/config") + + +@pytest.mark.usefixtures("connection_setup") +def test_role_present(vault_db, roleargs): + ret = vault_db.role_present("testrole", **roleargs) + assert ret.result + assert ret.changes + assert "created" in ret.changes + assert ret.changes["created"] == "testrole" + assert "testrole" in vault_list("database/roles") + + +@pytest.mark.usefixtures("roles_setup") +def test_role_present_no_changes(vault_db, roleargs): + ret = vault_db.role_present("testrole", **roleargs) + assert ret.result + assert not ret.changes + + +@pytest.mark.usefixtures("roles_setup") +def test_role_present_no_changes_with_time_string(vault_db, roleargs): + roleargs["default_ttl"] = "1h" + ret = vault_db.role_present("testrole", **roleargs) + assert ret.result + assert not ret.changes + + +@pytest.mark.usefixtures("roles_setup") +def test_role_present_param_change(vault_db, roleargs): + roleargs["default_ttl"] = 1337 + ret = vault_db.role_present("testrole", **roleargs) + assert ret.result + assert ret.changes + assert "default_ttl" in ret.changes + assert vault_read("database/roles/testrole")["data"]["default_ttl"] == 1337 + + +@pytest.mark.usefixtures("connection_setup") +def test_role_present_test_mode(vault_db, roleargs): + ret = vault_db.role_present("testrole", test=True, **roleargs) + assert ret.result is None + assert ret.changes + assert "created" in ret.changes + assert ret.changes["created"] == "testrole" + assert "testrole" not in vault_list("database/roles") + + +@pytest.mark.usefixtures("connection_setup") +def test_static_role_present(vault_db, roleargs_static): + ret = vault_db.static_role_present("teststaticrole", **roleargs_static) + assert ret.result + assert ret.changes + assert "created" in ret.changes + assert ret.changes["created"] == "teststaticrole" + assert "teststaticrole" in vault_list("database/static-roles") + + +@pytest.mark.usefixtures("role_static_setup") +def test_static_role_present_no_changes(vault_db, roleargs_static): + ret = vault_db.static_role_present("teststaticrole", **roleargs_static) + assert ret.result + assert not ret.changes + + +@pytest.mark.usefixtures("role_static_setup") +def test_static_role_present_param_change(vault_db, roleargs_static): + roleargs_static["rotation_period"] = 1337 + ret = vault_db.static_role_present("teststaticrole", **roleargs_static) + assert ret.result + assert ret.changes + assert "rotation_period" in ret.changes + assert vault_read("database/static-roles/teststaticrole")["data"]["rotation_period"] == 1337 + + +@pytest.mark.usefixtures("connection_setup") +def test_static_role_present_test_mode(vault_db, roleargs_static): + ret = vault_db.static_role_present("teststaticrole", test=True, **roleargs_static) + assert ret.result is None + assert ret.changes + assert "created" in ret.changes + assert ret.changes["created"] == "teststaticrole" + assert "teststaticrole" not in vault_list("database/static-roles") + + +@pytest.mark.usefixtures("roles_setup") +def test_role_absent(vault_db): + ret = vault_db.role_absent("testrole") + assert ret.result + assert ret.changes + assert "deleted" in ret.changes + assert ret.changes["deleted"] == "testrole" + assert "testrole" not in vault_list("database/roles") + + +@pytest.mark.usefixtures("role_static_setup") +def test_role_absent_static(vault_db): + ret = vault_db.role_absent("teststaticrole", static=True) + assert ret.result + assert ret.changes + assert "deleted" in ret.changes + assert ret.changes["deleted"] == "teststaticrole" + assert "teststaticrole" not in vault_list("database/static-roles") + + +def test_role_absent_no_changes(vault_db): + ret = vault_db.role_absent("testrole") + assert ret.result + assert not ret.changes + + +@pytest.mark.usefixtures("roles_setup") +def test_role_absent_test_mode(vault_db): + ret = vault_db.role_absent("testrole", test=True) + assert ret.result is None + assert ret.changes + assert "deleted" in ret.changes + assert ret.changes["deleted"] == "testrole" + assert "testrole" in vault_list("database/roles") diff --git a/tests/support/mysql.py b/tests/support/mysql.py new file mode 100644 index 00000000..7ed347ab --- /dev/null +++ b/tests/support/mysql.py @@ -0,0 +1,195 @@ +""" +This is copied from Salt's testsuite at tests/support/pytest/mysql.py. +""" +import logging +import time + +import attr +import pytest +from pytestskipmarkers.utils import platform +from saltfactories.utils import random_string + +# This `pytest.importorskip` here actually works because this module +# is imported into test modules, otherwise, the skipping would just fail +pytest.importorskip("docker") +import docker.errors # isort:skip + +log = logging.getLogger(__name__) + + +@attr.s(kw_only=True, slots=True) +class MySQLImage: + name = attr.ib() + tag = attr.ib() + container_id = attr.ib() + + def __str__(self): + return f"{self.name}:{self.tag}" + + +@attr.s(kw_only=True, slots=True) +class MySQLCombo: + mysql_name = attr.ib() + mysql_version = attr.ib() + mysql_port = attr.ib(default=None) + mysql_host = attr.ib(default="%") + mysql_user = attr.ib() + mysql_passwd = attr.ib() + mysql_database = attr.ib(default=None) + mysql_root_user = attr.ib(default="root") + mysql_root_passwd = attr.ib() + container = attr.ib(default=None) + container_id = attr.ib() + + @container_id.default + def _default_container_id(self): + return random_string( + "{}-{}-".format( # pylint: disable=consider-using-f-string + self.mysql_name.replace("/", "-"), + self.mysql_version, + ) + ) + + @mysql_root_passwd.default + def _default_mysql_root_user_passwd(self): + return self.mysql_passwd + + def get_credentials(self, **kwargs): + return { + "connection_user": kwargs.get("connection_user") or self.mysql_root_user, + "connection_pass": kwargs.get("connection_pass") or self.mysql_root_passwd, + "connection_db": kwargs.get("connection_db") or "mysql", + "connection_port": kwargs.get("connection_port") or self.mysql_port, + } + + +def get_test_versions(): + test_versions = [] + name = "mysql-server" + for version in ("5.5", "5.6", "5.7", "8.0"): + test_versions.append( + MySQLImage( + name=name, + tag=version, + container_id=random_string(f"mysql-{version}-"), + ) + ) + name = "mariadb" + for version in ("10.3", "10.4", "10.5"): + test_versions.append( + MySQLImage( + name=name, + tag=version, + container_id=random_string(f"mariadb-{version}-"), + ) + ) + name = "percona" + for version in ("5.6", "5.7", "8.0"): + test_versions.append( + MySQLImage( + name=name, + tag=version, + container_id=random_string(f"percona-{version}-"), + ) + ) + return test_versions + + +def get_test_version_id(value): + return f"container={value}" + + +@pytest.fixture(scope="module", params=get_test_versions(), ids=get_test_version_id) +def mysql_image(request): + return request.param + + +@pytest.fixture(scope="module") +def create_mysql_combo(mysql_image): + if platform.is_fips_enabled(): + if mysql_image.name in ("mysql-server", "percona") and mysql_image.tag == "8.0": + pytest.skip(f"These tests fail on {mysql_image.name}:{mysql_image.tag}") + + return MySQLCombo( + mysql_name=mysql_image.name, + mysql_version=mysql_image.tag, + mysql_user="salt-mysql-user", + mysql_passwd="Pa55w0rd!", + container_id=mysql_image.container_id, + ) + + +@pytest.fixture(scope="module") +def mysql_combo(create_mysql_combo): + return create_mysql_combo + + +def check_container_started(timeout_at, container, combo): + sleeptime = 0.5 + while time.time() <= timeout_at: + try: + if not container.is_running(): + log.warning("%s is no longer running", container) + return False + ret = container.run( + "mysql", + f"--user={combo.mysql_user}", + f"--password={combo.mysql_passwd}", + "-e", + "SELECT 1", + ) + if ret.returncode == 0: + break + except docker.errors.APIError: + log.exception("Failed to run start check") + time.sleep(sleeptime) + sleeptime *= 2 + else: + return False + time.sleep(0.5) + return True + + +def set_container_name_before_start(container): + """ + This is useful if the container has to be restared and the old + container, under the same name was left running, but in a bad shape. + """ + container.name = random_string( + "{}-".format(container.name.rsplit("-", 1)[0]) # pylint: disable=consider-using-f-string + ) + container.display_name = None + return container + + +@pytest.fixture(scope="module") +def mysql_container(salt_factories, mysql_combo): + + container_environment = { + "MYSQL_ROOT_PASSWORD": mysql_combo.mysql_passwd, + "MYSQL_ROOT_HOST": mysql_combo.mysql_host, + "MYSQL_USER": mysql_combo.mysql_user, + "MYSQL_PASSWORD": mysql_combo.mysql_passwd, + } + if mysql_combo.mysql_database: + container_environment["MYSQL_DATABASE"] = mysql_combo.mysql_database + + container = salt_factories.get_container( + mysql_combo.container_id, + "ghcr.io/saltstack/salt-ci-containers/{}:{}".format( # pylint: disable=consider-using-f-string + mysql_combo.mysql_name, mysql_combo.mysql_version + ), + pull_before_start=True, + skip_on_pull_failure=True, + skip_if_docker_client_not_connectable=True, + container_run_kwargs={ + "ports": {"3306/tcp": None}, + "environment": container_environment, + }, + ) + container.before_start(set_container_name_before_start, container) + container.container_start_check(check_container_started, container, mysql_combo) + with container.started(): + mysql_combo.container = container + mysql_combo.mysql_port = container.get_host_port_binding(3306, protocol="tcp", ipv6=False) + yield mysql_combo diff --git a/tests/support/vault.py b/tests/support/vault.py index d915d354..efc8e468 100644 --- a/tests/support/vault.py +++ b/tests/support/vault.py @@ -216,9 +216,59 @@ def vault_delete_secret(path, metadata=False): return True +def vault_delete(path): + try: + ret = _vault_cmd(["delete", "-format=json", path]) + except RuntimeError as err: + pytest.fail(f"Failed to delete path at `{path}`: {err}") + try: + return json.loads(ret.stdout) or True + except json.decoder.JSONDecodeError: + return True + + def vault_list(path): try: ret = _vault_cmd(["list", "-format=json", path]) except RuntimeError: pytest.fail(f"Failed to list path at `{path}`") return json.loads(ret.stdout) + + +def vault_read(path): + try: + ret = _vault_cmd(["read", "-format=json", path]) + except RuntimeError as err: + pytest.fail(f"Failed to read path at `{path}`: {err}") + return json.loads(ret.stdout) + + +def vault_write(path, *args, **kwargs): + kwargs_ = [f"{k}={v}" for k, v in kwargs.items()] + cmd = ( + ["write", "-format=json"] + + (["-f"] if not (args or kwargs) else []) + + [path] + + list(args) + + kwargs_ + ) + try: + ret = _vault_cmd(cmd) + except RuntimeError as err: + pytest.fail(f"Failed to write to path at `{path}`: {err}") + try: + return json.loads(ret.stdout) or True + except json.decoder.JSONDecodeError: + return True + + +def vault_revoke(lease_id, prefix=False): + cmd = ["lease", "revoke"] + if prefix: + cmd += ["-prefix"] + cmd += [lease_id] + try: + _vault_cmd(cmd) + except RuntimeError as err: + pytest.fail(f"Failed to revoke lease `{lease_id}`: {err}") + return True