diff --git a/.travis.yml b/.travis.yml index d13a9cb..8ceaca2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,9 +29,11 @@ env: before_install: - docker pull quay.io/coreos/etcd:$ETCD_VER - - docker run -d -p 2379:2379 -p 2380:2380 --name etcd3_1 quay.io/coreos/etcd:$ETCD_VER etcd --name node1 --initial-advertise-peer-urls http://0.0.0.0:2380 --listen-peer-urls http://0.0.0.0:2380 --advertise-client-urls http://0.0.0.0:2379 --listen-client-urls http://0.0.0.0:2379 --initial-cluster node1=http://0.0.0.0:2380 - - docker ps - - sudo docker cp etcd3_1:/usr/local/bin/etcdctl /usr/bin/etcdctl +# - docker run -d -p 2379:2379 -p 2380:2380 --name etcd3_1 quay.io/coreos/etcd:$ETCD_VER etcd --name node1 --initial-advertise-peer-urls http://0.0.0.0:2380 --listen-peer-urls http://0.0.0.0:2380 --advertise-client-urls http://0.0.0.0:2379 --listen-client-urls http://0.0.0.0:2379 --initial-cluster node1=http://0.0.0.0:2380 +# - docker ps + - docker create --name tmp quay.io/coreos/etcd:$ETCD_VER + - docker cp tmp:/usr/local/bin/etcdctl /usr/bin/etcdctl + - docker rm tmp - which etcdctl # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors diff --git a/etcd3/client.py b/etcd3/client.py index 6a0f7ef..8941225 100644 --- a/etcd3/client.py +++ b/etcd3/client.py @@ -3,7 +3,6 @@ """ import json - import requests import six from six.moves import urllib_parse @@ -189,7 +188,8 @@ def call_rpc(self, method, data=None, stream=False, encode=True, raw=False, **kw if self.token: kwargs.setdefault('headers', {}).setdefault('authorization', self.token) kwargs.setdefault('headers', {}).setdefault('user_agent', self.user_agent) - kwargs.setdefault('headers', {}).update(self.headers) + for k, v in self.headers.items(): + kwargs.setdefault('headers', {}).setdefault(k, v) if isinstance(data, dict): if encode: data = self._encodeRPCRequest(method, data) diff --git a/etcd3/stateful/watch.py b/etcd3/stateful/watch.py index b4122ee..068cffe 100644 --- a/etcd3/stateful/watch.py +++ b/etcd3/stateful/watch.py @@ -1,11 +1,11 @@ +import time + import logging import re +import six import socket import threading -import time from collections import deque - -import six from requests import ConnectionError from requests.exceptions import ChunkedEncodingError @@ -110,6 +110,7 @@ def __init__(self, client, max_retries=-1, key=None, range_end=None, start_revis """ self.client = client self.revision = None + self.watch_id = None self.retries = 0 self.errors = deque(maxlen=20) if max_retries == -1: @@ -163,9 +164,10 @@ def request_create(self): def request_cancel(self): # pragma: no cover """ - Cancel the watcher [Not Implemented because of etcd3] + Cancel the watcher [Not Implemented because of etcd3 returns no watch_id] """ - pass + if self.watch_id: + return self.client.watch_cancel(watch_id=self.watch_id) def get_filter(self, filter): """ @@ -301,6 +303,7 @@ def stop(self): if not self._resp or (self._resp and self._resp.raw.closed): return try: + self.request_cancel() s = socket.fromfd(self._resp.raw._fp.fileno(), socket.AF_INET, socket.SOCK_STREAM) s.shutdown(socket.SHUT_RDWR) s.close() @@ -368,6 +371,7 @@ def __iter__(self): if 'created' in r: log.debug("watch request created") self.start_revision = r.header.revision + self.watch_id = r.watch_id if 'events' in r: for event in r.events: yield Event(event) diff --git a/tests/envs.py b/tests/envs.py index 2420689..5fef255 100644 --- a/tests/envs.py +++ b/tests/envs.py @@ -1,7 +1,13 @@ +import logging import os - from six.moves.urllib_parse import urlparse +logging.basicConfig(format='%(name)s %(levelname)s - %(message)s') +log = logging.getLogger() +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler() +log.addHandler(handler) + ETCD_ENDPOINT = os.getenv('ETCD_ENDPOINT') or 'http://localhost:2379' _url = urlparse(ETCD_ENDPOINT) @@ -9,6 +15,6 @@ host, port = _url.netloc.split(':') -ETCD_VER = os.getenv('ETCD_VER') or 'v3.3' +ETCD_VER = os.getenv('ETCD_VER') or 'v3.3.0' ETCD_IMG = 'quay.io/coreos/etcd:' + ETCD_VER diff --git a/tests/test_auth_apis.py b/tests/test_auth_apis.py index 7c76d65..8788970 100644 --- a/tests/test_auth_apis.py +++ b/tests/test_auth_apis.py @@ -1,4 +1,6 @@ +import logging import pytest +import re from etcd3.client import Client from etcd3.errors import ErrAuthNotEnabled @@ -8,7 +10,7 @@ from etcd3.models import authpbPermissionType from etcd3.utils import incr_last_byte from tests.docker_cli import docker_run_etcd_main -from .envs import protocol, host +from .envs import protocol, host, ETCD_VER from .etcd_go_cli import etcdctl, NO_ETCD_SERVICE @@ -43,9 +45,10 @@ def enable_auth(): # pragma: no cover @pytest.mark.skipif(NO_ETCD_SERVICE, reason="no etcd service available") +# @pytest.mark.skipif(re.match(r'v3\.[0-2]\.{0,1}', ETCD_VER), reason="etcd < v3.3.0 does not support auth header") def test_auth_flow(client, request): teardown_auth(client) - request.addfinalizer(lambda:teardown_auth(client)) + request.addfinalizer(lambda: teardown_auth(client)) # test error with pytest.raises(ErrRootUserNotExist): @@ -82,26 +85,29 @@ def test_auth_flow(client, request): assert len(r.roles) == 1 assert r.roles[0] == 'root' - # test auth enable - client.auth_enable() - r = client.authenticate('root', 'root') - assert r.token - - # test client auth - client.auth('root', 'root') - assert client.token - assert client.user_get('root') - - # test user change password - client.user_change_password('root', 'changed') - client.auth('root', 'changed') - client.user_change_password('root', 'root') - client.auth('root', 'root') - - # test auth disable - client.auth_disable() - with pytest.raises(ErrAuthNotEnabled): - client.authenticate('root', 'root') + if re.match(r'v3\.[0-2]\.{0,1}', ETCD_VER): + logging.info('skipping tests of apis with auth enabled, cause etcd < v3.3.0 does not support auth header') + else: + # test auth enable + client.auth_enable() + r = client.authenticate('root', 'root') + assert r.token + + # test client auth + client.auth('root', 'root') + assert client.token + assert client.user_get('root') + + # test user change password + client.user_change_password('root', 'changed') + client.auth('root', 'changed') + client.user_change_password('root', 'root') + client.auth('root', 'root') + + # test auth disable + client.auth_disable() + with pytest.raises(ErrAuthNotEnabled): + client.authenticate('root', 'root') # test user revoke role client.user_revoke_role('root', 'root') diff --git a/tests/test_py3/test_aio_auth_apis.py b/tests/test_py3/test_aio_auth_apis.py index 6e8d229..cdb82c7 100644 --- a/tests/test_py3/test_aio_auth_apis.py +++ b/tests/test_py3/test_aio_auth_apis.py @@ -1,9 +1,12 @@ import asyncio +import logging import pytest +import re from etcd3 import AioClient -from ..envs import protocol, host, port +from tests.docker_cli import docker_run_etcd_main +from ..envs import protocol, host, ETCD_VER from ..etcd_go_cli import NO_ETCD_SERVICE from ..etcd_go_cli import etcdctl @@ -24,8 +27,8 @@ async def aio_client(event_loop, request): """ init Etcd3Client, close its connection-pool when teardown """ - - c = AioClient(host, port, protocol) + _, p, _ = docker_run_etcd_main() + c = AioClient(host, p, protocol) def teardown(): async def _t(): @@ -37,6 +40,7 @@ async def _t(): request.addfinalizer(teardown) return c + def teardown_auth(): # pragma: no cover """ disable auth, delete all users and roles @@ -55,14 +59,18 @@ def enable_auth(): # pragma: no cover etcdctl('user grant root root') etcdctl('auth enable') + @pytest.mark.skipif(NO_ETCD_SERVICE, reason="no etcd service available") @pytest.mark.asyncio async def test_async_client_auth(aio_client, request): + teardown_auth() enable_auth() request.addfinalizer(teardown_auth) # test client auth await aio_client.auth('root', 'root') assert aio_client.token - assert aio_client.user_get('root') - + if re.match(r'v3\.[0-2]\.{0,1}', ETCD_VER): + logging.info('skipping tests of apis with auth enabled, cause etcd < v3.3.0 does not support auth header') + else: + assert await aio_client.user_get('root') diff --git a/tests/test_py3/test_aio_client.py b/tests/test_py3/test_aio_client.py index ad1a3d3..f225fc0 100644 --- a/tests/test_py3/test_aio_client.py +++ b/tests/test_py3/test_aio_client.py @@ -4,8 +4,9 @@ import pytest from etcd3 import AioClient -from ..docker_cli import docker_rm_etcd_ssl, docker_run_etcd_ssl, CERT_PATH, KEY_PATH, CA_PATH, NO_DOCKER_SERVICE -from ..envs import protocol, host, port +from ..docker_cli import docker_rm_etcd_ssl, docker_run_etcd_ssl, CERT_PATH, KEY_PATH, CA_PATH, NO_DOCKER_SERVICE, \ + docker_run_etcd_main +from ..envs import protocol, host from ..etcd_go_cli import NO_ETCD_SERVICE from ..etcd_go_cli import etcdctl @@ -27,7 +28,8 @@ async def aio_client(event_loop, request): init Etcd3Client, close its connection-pool when teardown """ - c = AioClient(host, port, protocol) + _, p, _ = docker_run_etcd_main() + c = AioClient(host, p, protocol) def teardown(): async def _t():