Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Port #52001 to master #56841

Closed
wants to merge 4 commits into from
Closed

Port #52001 to master #56841

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4106121
Add tests for syncing extmods
waynew Feb 23, 2021
f2d27b4
extmods: add utils directories in sys.path
aplanas Mar 6, 2019
14d4b64
drop py2 code
waynew Feb 23, 2021
5c9115f
Fix potential bug
waynew Feb 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 20 additions & 15 deletions salt/utils/extmods.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,19 @@
# -*- coding: utf-8 -*-
"""
Functions used to sync external modules
"""
from __future__ import absolute_import, print_function, unicode_literals

# Import Python libs
import logging
import os
import pathlib
import shutil
import sys

# Import salt libs
import salt.fileclient
import salt.utils.files
import salt.utils.hashutils
import salt.utils.path
import salt.utils.url

# Import 3rd-party libs
from salt.ext import six

log = logging.getLogger(__name__)


Expand Down Expand Up @@ -48,7 +43,7 @@ def sync(opts, form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None)

if extmod_whitelist is None:
extmod_whitelist = opts["extmod_whitelist"]
elif isinstance(extmod_whitelist, six.string_types):
elif isinstance(extmod_whitelist, str):
extmod_whitelist = {form: extmod_whitelist.split(",")}
elif not isinstance(extmod_whitelist, dict):
log.error(
Expand All @@ -57,27 +52,27 @@ def sync(opts, form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None)

if extmod_blacklist is None:
extmod_blacklist = opts["extmod_blacklist"]
elif isinstance(extmod_blacklist, six.string_types):
elif isinstance(extmod_blacklist, str):
extmod_blacklist = {form: extmod_blacklist.split(",")}
elif not isinstance(extmod_blacklist, dict):
log.error(
"extmod_blacklist must be a string or dictionary: %s", extmod_blacklist
)

if isinstance(saltenv, six.string_types):
if isinstance(saltenv, str):
saltenv = saltenv.split(",")
ret = []
remote = set()
source = salt.utils.url.create("_" + form)
mod_dir = os.path.join(opts["extension_modules"], "{0}".format(form))
mod_dir = os.path.join(opts["extension_modules"], "{}".format(form))
touched = False
with salt.utils.files.set_umask(0o077):
try:
if not os.path.isdir(mod_dir):
log.info("Creating module dir '%s'", mod_dir)
try:
os.makedirs(mod_dir)
except (IOError, OSError):
except OSError:
log.error(
"Cannot create cache module directory %s. Check "
"permissions.",
Expand All @@ -99,7 +94,7 @@ def sync(opts, form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None)
)
)
local_cache_dir = os.path.join(
opts["cachedir"], "files", sub_env, "_{0}".format(form)
opts["cachedir"], "files", sub_env, "_{}".format(form)
)
log.debug("Local cache dir: '%s'", local_cache_dir)
for fn_ in cache:
Expand Down Expand Up @@ -128,13 +123,23 @@ def sync(opts, form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None)
if src_digest != dst_digest:
# The downloaded file differs, replace!
shutil.copyfile(fn_, dest)
ret.append("{0}.{1}".format(form, relname))
ret.append("{}.{}".format(form, relname))
else:
dest_dir = os.path.dirname(dest)
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
shutil.copyfile(fn_, dest)
ret.append("{0}.{1}".format(form, relname))
ret.append("{}.{}".format(form, relname))
# If the synchronized module is an utils
# directory, we add it to sys.path
for util_dir in opts["utils_dirs"]:
util_parts = pathlib.Path(util_dir).parts
mod_parts = pathlib.Path(mod_dir).parts
if (
util_parts == mod_parts[-len(util_parts) :]
and mod_dir not in sys.path
):
sys.path.append(mod_dir)

touched = bool(ret)
if opts["clean_dynamic_modules"] is True:
Expand Down
131 changes: 131 additions & 0 deletions tests/pytests/unit/utils/test_extmods.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
import os

import pytest
import salt.utils.extmods as extmods
from tests.support.mock import patch


@pytest.mark.parametrize(
"utils_dirs",
[["blerp"], ["/bang/fnord/blerp", "d/bang/fnord/blerp", "bang/fnord/blerp"]],
)
def test_when_utils_dirs_in_opts_and_mod_dir_ends_with_any_dir_in_utils_and_mod_dir_not_on_sys_path_then_mod_dir_should_be_added_to_sys_path(
utils_dirs,
):

dir_to_sync = "blerp"
extension_modules = "fnord/bang/fnord"
expected_path = [os.path.join(extension_modules, dir_to_sync)]
fake_path = []

with patch(
"salt.fileclient.get_file_client", autospec=True
) as fake_fileclient, patch("shutil.copyfile", autospec=True), patch(
"sys.path", fake_path
):
fake_fileclient.return_value.cache_dir.return_value = ["something_good"]
extmods.sync(
opts={
"utils_dirs": utils_dirs,
"extension_modules": extension_modules,
"extmod_whitelist": None,
"extmod_blacklist": None,
"cachedir": "",
"clean_dynamic_modules": False,
},
form=dir_to_sync,
)

assert fake_path == expected_path


def test_when_utils_dirs_is_empty_then_mod_dir_should_not_be_added_to_path():
expected_path = []
fake_path = []
empty_utils_dirs = []

with patch(
"salt.fileclient.get_file_client", autospec=True
) as fake_fileclient, patch("shutil.copyfile", autospec=True), patch(
"sys.path", fake_path
):
fake_fileclient.return_value.cache_dir.return_value = ["something_good"]
extmods.sync(
opts={
"utils_dirs": empty_utils_dirs,
"extmod_whitelist": None,
"extmod_blacklist": None,
"extension_modules": "fnord",
"cachedir": "",
"clean_dynamic_modules": False,
},
form="blerp",
)

assert (
fake_path == expected_path
), "mod_dir was added to sys.path when it should not have been."


@pytest.mark.parametrize(
"utils_dirs",
[
["mid"],
["start/mid"],
["start/m/end", "id/end", "d/end", "/end", "/mid/end", "t/mid/end"],
],
)
def test_when_utils_dirs_but_mod_dir_is_not_parent_of_any_util_dir_then_mod_dir_should_not_be_added_to_path(
utils_dirs,
):
dir_to_sync = "end"
extension_modules = "start/mid"
expected_path = []
fake_path = []

with patch(
"salt.fileclient.get_file_client", autospec=True
) as fake_fileclient, patch("shutil.copyfile", autospec=True), patch(
"sys.path", fake_path
):
fake_fileclient.return_value.cache_dir.return_value = ["something_good"]
extmods.sync(
opts={
"utils_dirs": utils_dirs,
"extension_modules": extension_modules,
"extmod_whitelist": None,
"extmod_blacklist": None,
"cachedir": "",
"clean_dynamic_modules": False,
},
form=dir_to_sync,
)

assert fake_path == expected_path


def test_when_mod_dir_already_on_path_it_should_not_be_added_to_path():
dir_to_sync = "blerp"
extension_modules = "fnord/bang/fnord"
expected_path = [os.path.join(extension_modules, dir_to_sync)]
fake_path = expected_path[:]

with patch(
"salt.fileclient.get_file_client", autospec=True
) as fake_fileclient, patch("shutil.copyfile", autospec=True), patch(
"sys.path", fake_path
):
fake_fileclient.return_value.cache_dir.return_value = ["something_good"]
extmods.sync(
opts={
"utils_dirs": ["blerp", "blerp", "blerp"],
"extension_modules": extension_modules,
"extmod_whitelist": None,
"extmod_blacklist": None,
"cachedir": "",
"clean_dynamic_modules": False,
},
form=dir_to_sync,
)

assert fake_path == expected_path