feature(hydra): move to formal python:3.8 image

.pre-commit-config.yaml

@@ -38,7 +38,7 @@ repos:
         additional_dependencies:
             - 'requests==2.20.0'
             - 'fabric==2.4.0'
-            - 'PyYAML==3.11'
+            - 'PyYAML==5.1'
             - 'boto3==1.13.5'
             - 'boto3-stubs[s3,ec2,dynamodb,pricing]==1.13.5'
             - 'apache-libcloud==2.6.0'
@@ -52,7 +52,6 @@ repos:
             - 'click==7.0'
             - 'click-completion==0.5.0'
             - 'PTable==0.9.2'
-            - 'backports.datetime-timestamp==1.2'
             - 'pytest==4.6.4'
             - 'selenium==3.141.0'
             - 'mysql-connector-python==8.0.19'

README.rst

@@ -104,6 +104,19 @@ To run SCT tests locally run following::
     sudo ./install-prereqs.sh
     ./get-qa-ssh-keys.sh
 
+    # install python3.8 via pyenv
+    curl https://pyenv.run | bash
+    exec $SHELL
+    # go to: https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites
+    # and follow the instructions for your distribution, to install the prerequisites
+    # for compiling python from source
+    pyenv install 3.8.3
+
+    # create a virtualenv for SCT
+    pyenv virtualenv 3.8.3 sct38
+    pyenv activate sct38
+    pip install -r requirements-python.txt
+
 Run a test
 ----------
 

docker/env/Dockerfile

@@ -1,15 +1,18 @@
-FROM centos:7
+FROM python:3.8
 
-ADD install-prereqs.sh install-prereqs.sh
-RUN ./install-prereqs.sh docker
-RUN yum update -y \
-    && yum install -y https://repo.ius.io/ius-release-el7.rpm \
-    && yum install -y python36u python36u-libs python36u-devel python36u-pip
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+    apt-get install -y git openssl iproute2 sudo apt-transport-https \
+    ca-certificates curl gnupg2 software-properties-common rsync
+
+# Install Docker cli only
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add - && \
+    DEBIAN_FRONTEND=noninteractive add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \
+    DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y docker-ce-cli  && \
+    sudo groupadd docker && \
+    sudo usermod -aG docker `whoami`
 
 ADD requirements-python.txt requirements-python.txt
-RUN pip3 install setuptools==40.8.0 pip==19.1.1
+RUN pip3 install setuptools==46.4.0 pip==20.1.1
 RUN pip3 install -r requirements-python.txt
 
-ENV LANG en_US.UTF-8
-ENV LC_ALL en_US.UTF-8
 ENV PYTHONWARNINGS ignore:unclosed

docker/env/build_n_push.sh

@@ -6,7 +6,6 @@ VERSION=v$(cat ${DOCKER_ENV_DIR}/version)
 
 SCT_DIR=$(dirname $(dirname ${DOCKER_ENV_DIR}))
 PY_PREREQS_FILE=requirements-python.txt
-CENTOS_PREREQS_FILE=install-prereqs.sh
 
 function docker_tag_exists() {
     echo "Contacting Docker Hub to check if Hydra image ${VERSION} already exists..."
@@ -26,9 +25,8 @@ else
     echo "Hydra image with version $VERSION not found locally. Building..."
     cd "${DOCKER_ENV_DIR}"
     cp -f ${SCT_DIR}/${PY_PREREQS_FILE} .
-    cp -f ${SCT_DIR}/${CENTOS_PREREQS_FILE} .
     docker build --network=host -t scylladb/hydra:${VERSION} .
-    rm -f ${PY_PREREQS_FILE} ${CENTOS_PREREQS_FILE}
+    rm -f ${PY_PREREQS_FILE}
     cd -
     docker login
     echo "Tagging and pushing..."

docker/env/version

@@ -1 +1 @@
-0.66
+0.68

sdcm/cluster.py

@@ -29,6 +29,7 @@
 from typing import List, Optional, Dict
 from textwrap import dedent
 from datetime import datetime
+from functools import cached_property
 
 from invoke.exceptions import UnexpectedExit, Failure, CommandTimedOut
 import yaml
@@ -51,7 +52,7 @@
 
 from sdcm.utils.health_checker import check_nodes_status, check_node_status_in_gossip_and_nodetool_status, \
     check_schema_version, check_nulls_in_peers
-from sdcm.utils.decorators import cached_property, retrying, log_run_info
+from sdcm.utils.decorators import retrying, log_run_info
 from sdcm.utils.get_username import get_username
 from sdcm.utils.remotewebbrowser import WebDriverContainerMixin
 from sdcm.utils.version_utils import SCYLLA_VERSION_RE, get_gemini_version
@@ -3738,13 +3739,11 @@ def node_setup(self, node, verbose=False, db_node_address=None, **kwargs):  # py
         node.download_scylla_repo(scylla_repo_loader)
         if node.is_rhel_like():
             node.remoter.run('sudo yum install -y {}-tools'.format(node.scylla_pkg()))
-            node.remoter.run('sudo yum install -y screen')
         else:
             node.remoter.run('sudo apt-get update')
             node.remoter.run('sudo apt-get install -y -o Dpkg::Options::="--force-confdef"'
                              ' -o Dpkg::Options::="--force-confold" --force-yes'
                              ' --allow-unauthenticated {}-tools'.format(node.scylla_pkg()))
-            node.remoter.run('sudo apt-get install -y screen')
 
         if db_node_address is not None:
             node.remoter.run("echo 'export DB_ADDRESS=%s' >> $HOME/.bashrc" % db_node_address)
@@ -4161,10 +4160,6 @@ def node_setup(self, node, **kwargs):  # pylint: disable=unused-argument
         # since monitoring node is started last (after db nodes and loader) we can't actually set the timeout
         # for starting the alert manager thread (since it depends on DB cluster size and num of loaders)
         node.start_alert_manager_thread()  # remove when start task threads will be started after node setup
-        if node.is_rhel_like():
-            node.remoter.run('sudo yum install screen -y')
-        else:
-            node.remoter.run('sudo apt-get install screen -y')
         if self.params.get("use_mgmt", default=None):
             self.install_scylla_manager(node, auth_token=self.mgmt_auth_token)
 
@@ -4414,6 +4409,7 @@ def reconfigure_scylla_monitoring(self):
             configure_script = dedent("""
                         cd {0.monitor_install_path}
                         mkdir -p {0.monitoring_conf_dir}
+                        export PATH=/usr/local/bin:$PATH  # hack to enable running on docker
                         python3 genconfig.py -s -n -d {0.monitoring_conf_dir} {0._monitoring_targets}
                     """.format(self))
             node.remoter.run("sudo bash -ce '%s'" % configure_script, verbose=True)

sdcm/cluster_aws.py

@@ -12,6 +12,7 @@
 from textwrap import dedent
 from datetime import datetime
 from distutils.version import LooseVersion  # pylint: disable=no-name-in-module,import-error
+from functools import cached_property
 
 import yaml
 from botocore.exceptions import WaiterError, ClientError
@@ -22,7 +23,7 @@
 from sdcm import ec2_client
 from sdcm.cluster import INSTANCE_PROVISION_ON_DEMAND
 from sdcm.utils.common import list_instances_aws, get_ami_tags
-from sdcm.utils.decorators import retrying, cached_property
+from sdcm.utils.decorators import retrying
 from sdcm.sct_events import SpotTerminationEvent, DbEventsFilter
 from sdcm import wait
 from sdcm.remote import LocalCmdRunner, NETWORK_EXCEPTIONS

sdcm/cluster_docker.py

@@ -18,13 +18,14 @@
 import re
 import logging
 from typing import Optional, Union, Dict
+from functools import cached_property
 
 from sdcm import cluster
 from sdcm.remote import LOCALRUNNER
 from sdcm.utils.docker_utils import get_docker_bridge_gateway, Container, ContainerManager, DockerException
-from sdcm.utils.decorators import cached_property
 from sdcm.utils.health_checker import check_nodes_status
 
+
 DEFAULT_SCYLLA_DB_IMAGE = "scylladb/scylla-nightly"
 DEFAULT_SCYLLA_DB_IMAGE_TAG = "latest"
 AIO_MAX_NR_RECOMMENDED_VALUE = 1048576

sdcm/cluster_gce.py

@@ -16,13 +16,13 @@
 import logging
 from textwrap import dedent
 from typing import Dict
+from functools import cached_property
 
 from libcloud.common.google import GoogleBaseError, ResourceNotFoundError
 
 from sdcm import cluster
 from sdcm.sct_events import SpotTerminationEvent
 from sdcm.utils.common import list_instances_gce, gce_meta_to_dict
-from sdcm.utils.decorators import cached_property
 
 
 SPOT_TERMINATION_CHECK_DELAY = 1

sdcm/logcollector.py

@@ -9,8 +9,9 @@
 import zipfile
 import fnmatch
 import traceback
-import requests
+from functools import cached_property
 
+import requests
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.common.by import By
@@ -19,7 +20,7 @@
                                ParallelObject, remove_files, get_builder_by_test_id,
                                get_testrun_dir, search_test_id_in_latest, filter_aws_instances_by_type,
                                filter_gce_instances_by_type, get_sct_root_path)
-from sdcm.utils.decorators import retrying, cached_property
+from sdcm.utils.decorators import retrying
 from sdcm.utils.get_username import get_username
 from sdcm.db_stats import PrometheusDBStats
 from sdcm.remote import RemoteCmdRunner, LocalCmdRunner

sdcm/send_email.py

@@ -7,12 +7,12 @@
 from email.mime.application import MIMEApplication
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
+from functools import cached_property
 
 import jinja2
 
 from sdcm.keystore import KeyStore
 from sdcm.utils.common import list_instances_gce, list_instances_aws, list_resources_docker
-from sdcm.utils.decorators import cached_property
 
 LOGGER = logging.getLogger(__name__)
 

sdcm/utils/auto_ssh.py

@@ -12,8 +12,7 @@
 # Copyright (c) 2020 ScyllaDB
 
 import os
-
-from sdcm.utils.decorators import cached_property
+from functools import cached_property
 
 
 AUTO_SSH_IMAGE = "jnovack/autossh:latest"

sdcm/utils/cloud_monitor/resources/instances.py

@@ -1,8 +1,9 @@
 from logging import getLogger
+from datetime import datetime
+
 from sdcm.utils.cloud_monitor.common import InstanceLifecycle, NA
 from sdcm.utils.cloud_monitor.resources import CloudInstance, CloudResources
 from sdcm.utils.common import aws_tags_to_dict, gce_meta_to_dict, list_instances_aws, list_instances_gce
-from sdcm.utils.fromisoformat import datetime_from_isoformat
 from sdcm.utils.pricing import AWSPricing, GCEPricing
 
 
@@ -47,7 +48,7 @@ def __init__(self, instance):
             lifecycle=InstanceLifecycle.SPOT if is_preemptible else InstanceLifecycle.ON_DEMAND,
             instance_type=instance.size,
             owner=tags.get("RunByUser", NA) if tags else NA,
-            create_time=datetime_from_isoformat(instance.extra['creationTimestamp']),
+            create_time=datetime.fromisoformat(instance.extra['creationTimestamp']),
             keep=self.get_keep_alive_gce_instance(instance),
         )
 

sdcm/utils/common.py

@@ -1088,8 +1088,10 @@ def get_free_port():
 
 
 def get_my_ip():
-    hostname = socket.gethostname()
-    ip = socket.gethostbyname(hostname)
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.connect(("8.8.8.8", 80))
+    ip = sock.getsockname()[0]
+    sock.close()
     return ip
 
 

sdcm/utils/decorators.py

@@ -15,74 +15,12 @@
 import time
 import logging
 import datetime
-import threading
 from functools import wraps, partial
 
 
 LOGGER = logging.getLogger(__name__)
 
 
-try:
-    from functools import cached_property  # pylint: disable=unused-import,ungrouped-imports
-except ImportError:
-    # Copy/pasted from https://github.com/python/cpython/blob/3.8/Lib/functools.py#L923
-
-    # TODO: remove this code after switching to Python 3.8+
-
-    ################################################################################
-    # cached_property() - computed once per instance, cached as attribute
-    ################################################################################
-
-    _NOT_FOUND = object()
-
-    class cached_property:  # pylint: disable=invalid-name,too-few-public-methods
-        def __init__(self, func):
-            self.func = func
-            self.attrname = None
-            self.__doc__ = func.__doc__
-            self.lock = threading.RLock()
-
-        def __set_name__(self, owner, name):
-            if self.attrname is None:
-                self.attrname = name
-            elif name != self.attrname:
-                raise TypeError(
-                    "Cannot assign the same cached_property to two different names "
-                    f"({self.attrname!r} and {name!r})."
-                )
-
-        def __get__(self, instance, owner=None):
-            if instance is None:
-                return self
-            if self.attrname is None:
-                raise TypeError(
-                    "Cannot use cached_property instance without calling __set_name__ on it.")
-            try:
-                cache = instance.__dict__
-            except AttributeError:  # not all objects have __dict__ (e.g. class defines slots)
-                msg = (
-                    f"No '__dict__' attribute on {type(instance).__name__!r} "
-                    f"instance to cache {self.attrname!r} property."
-                )
-                raise TypeError(msg) from None
-            val = cache.get(self.attrname, _NOT_FOUND)
-            if val is _NOT_FOUND:
-                with self.lock:
-                    # check if another thread filled cache while we awaited lock
-                    val = cache.get(self.attrname, _NOT_FOUND)
-                    if val is _NOT_FOUND:
-                        val = self.func(instance)
-                        try:
-                            cache[self.attrname] = val
-                        except TypeError:
-                            msg = (
-                                f"The '__dict__' attribute on {type(instance).__name__!r} instance "
-                                f"does not support item assignment for caching {self.attrname!r} property."
-                            )
-                            raise TypeError(msg) from None
-            return val
-
-
 class Retry(Exception):
     pass