Improve etcd comapabitity of multiple versions (#42)

* add some etcd versions to test comability

* skip auth header tests for etcd<v3.3.0

* update docs

.travis.yml

@@ -19,13 +19,21 @@ services:
 
 env:
   matrix:
-    - ETCD_VER=v3.3
+    - ETCD_VER=v3.2.20
+    - ETCD_VER=v3.3.0
+#    - ETCD_VER=v3.3.4
+#    - ETCD_VER=v3.3.7
+#    - ETCD_VER=v3.3.9
+    - ETCD_VER=v3.3.10
+
 
 before_install:
-  - docker pull quay.io/coreos/etcd:$ETCD_VER
-  - docker run -d -p 2379:2379 -p 2380:2380 --name etcd3_1 quay.io/coreos/etcd:$ETCD_VER etcd --name node1 --initial-advertise-peer-urls http://0.0.0.0:2380 --listen-peer-urls http://0.0.0.0:2380 --advertise-client-urls http://0.0.0.0:2379 --listen-client-urls http://0.0.0.0:2379 --initial-cluster node1=http://0.0.0.0:2380
-  - docker ps
-  - sudo docker cp etcd3_1:/usr/local/bin/etcdctl /usr/bin/etcdctl
+  - sudo docker pull quay.io/coreos/etcd:$ETCD_VER
+#  - docker run -d -p 2379:2379 -p 2380:2380 --name etcd3_1 quay.io/coreos/etcd:$ETCD_VER etcd --name node1 --initial-advertise-peer-urls http://0.0.0.0:2380 --listen-peer-urls http://0.0.0.0:2380 --advertise-client-urls http://0.0.0.0:2379 --listen-client-urls http://0.0.0.0:2379 --initial-cluster node1=http://0.0.0.0:2380
+#  - docker ps
+  - sudo docker create --name tmp quay.io/coreos/etcd:$ETCD_VER
+  - sudo docker cp tmp:/usr/local/bin/etcdctl /usr/bin/etcdctl && sudo chmod 755 /usr/bin/etcdctl
+  - sudo docker rm tmp
   - which etcdctl
 
 # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors

HISTORY.rst

@@ -2,7 +2,34 @@
 History
 =======
 
-0.0.1 (2018-01-26)
+0.1.5 (2018-07-4)
 ------------------
 
-* First release on PyPI.
+merge pull request `#34 <https://github.com/Revolution1/etcd3-py/pull/34>`_ enum34 only where it's needed
+
+0.1.4 (2018-03-30)
+------------------
+
+* better code quality
+* support etcd v3.2.2+
+
+
+0.1.3 (2018-03-21)
+------------------
+
+finished lock util
+
+0.1.2 (2018-03-20)
+------------------
+
+* Add more test
+* Add watcher, transaction and lease util
+
+You can try it at dev environment
+
+0.1.0 (2018-03-19)
+------------------
+
+* Implemented all APIs of etcd3's gRPC-JSON-Gateway
+
+* Stateful utils (Watcher Lease Lock Transaction) are in progress

README.md

@@ -151,8 +151,15 @@ docker run -d \
 --initial-cluster node1=http://${NODE1}:2380
 ```
 
+## FAQ
+
+**Q:** authentication seems not working? Try calling api of a auth-enabled etcd server returned error "ErrUserEmpty error:'etcdserver: user name is empty'"
+
+**A:** Take a look at [#41](https://github.com/Revolution1/etcd3-py/issues/41), currently etcd3-py dose not authenticate automatically, you need to call client.auth() by yourself.
+
 ## TODO
 
 - [ ] benchmark
 - [ ] python-etcd(etcd v2) compatible client
 - [ ] etcd browser
+- [ ] support etcd v3.4.x

etcd3/client.py

@@ -3,7 +3,6 @@
 """
 
 import json
-
 import requests
 import six
 from six.moves import urllib_parse
@@ -189,7 +188,8 @@ def call_rpc(self, method, data=None, stream=False, encode=True, raw=False, **kw
         if self.token:
             kwargs.setdefault('headers', {}).setdefault('authorization', self.token)
         kwargs.setdefault('headers', {}).setdefault('user_agent', self.user_agent)
-        kwargs.setdefault('headers', {}).update(self.headers)
+        for k, v in self.headers.items():
+            kwargs.setdefault('headers', {}).setdefault(k, v)
         if isinstance(data, dict):
             if encode:
                 data = self._encodeRPCRequest(method, data)

etcd3/stateful/watch.py

@@ -1,11 +1,11 @@
+import time
+
 import logging
 import re
+import six
 import socket
 import threading
-import time
 from collections import deque
-
-import six
 from requests import ConnectionError
 from requests.exceptions import ChunkedEncodingError
 
@@ -110,6 +110,7 @@ def __init__(self, client, max_retries=-1, key=None, range_end=None, start_revis
         """
         self.client = client
         self.revision = None
+        self.watch_id = None
         self.retries = 0
         self.errors = deque(maxlen=20)
         if max_retries == -1:
@@ -163,9 +164,10 @@ def request_create(self):
 
     def request_cancel(self):  # pragma: no cover
         """
-        Cancel the watcher [Not Implemented because of etcd3]
+        Cancel the watcher [Not Implemented because of etcd3 returns no watch_id]
         """
-        pass
+        if self.watch_id:
+            return self.client.watch_cancel(watch_id=self.watch_id)
 
     def get_filter(self, filter):
         """
@@ -301,6 +303,7 @@ def stop(self):
         if not self._resp or (self._resp and self._resp.raw.closed):
             return
         try:
+            self.request_cancel()
             s = socket.fromfd(self._resp.raw._fp.fileno(), socket.AF_INET, socket.SOCK_STREAM)
             s.shutdown(socket.SHUT_RDWR)
             s.close()
@@ -368,6 +371,7 @@ def __iter__(self):
                         if 'created' in r:
                             log.debug("watch request created")
                             self.start_revision = r.header.revision
+                            self.watch_id = r.watch_id
                         if 'events' in r:
                             for event in r.events:
                                 yield Event(event)

setup.py

@@ -68,7 +68,7 @@
     zip_safe=False,
     keywords='etcd3',
     classifiers=[
-        'Development Status :: 3 - Alpha',
+        'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: Apache Software License',
         'Natural Language :: English',

tests/envs.py

@@ -1,14 +1,20 @@
+import logging
 import os
-
 from six.moves.urllib_parse import urlparse
 
+logging.basicConfig(format='%(name)s %(levelname)s - %(message)s')
+log = logging.getLogger()
+log.setLevel(logging.DEBUG)
+handler = logging.StreamHandler()
+log.addHandler(handler)
+
 ETCD_ENDPOINT = os.getenv('ETCD_ENDPOINT') or 'http://localhost:2379'
 _url = urlparse(ETCD_ENDPOINT)
 
 protocol = _url.scheme
 
 host, port = _url.netloc.split(':')
 
-ETCD_VER = os.getenv('ETCD_VER') or 'v3.3'
+ETCD_VER = os.getenv('ETCD_VER') or 'v3.3.0'
 
 ETCD_IMG = 'quay.io/coreos/etcd:' + ETCD_VER

tests/test_auth_apis.py

@@ -1,4 +1,6 @@
+import logging
 import pytest
+import re
 
 from etcd3.client import Client
 from etcd3.errors import ErrAuthNotEnabled
@@ -8,7 +10,7 @@
 from etcd3.models import authpbPermissionType
 from etcd3.utils import incr_last_byte
 from tests.docker_cli import docker_run_etcd_main
-from .envs import protocol, host
+from .envs import protocol, host, ETCD_VER
 from .etcd_go_cli import etcdctl, NO_ETCD_SERVICE
 
 
@@ -43,9 +45,10 @@ def enable_auth():  # pragma: no cover
 
 
 @pytest.mark.skipif(NO_ETCD_SERVICE, reason="no etcd service available")
+# @pytest.mark.skipif(re.match(r'v3\.[0-2]\.{0,1}', ETCD_VER), reason="etcd < v3.3.0 does not support auth header")
 def test_auth_flow(client, request):
     teardown_auth(client)
-    request.addfinalizer(lambda:teardown_auth(client))
+    request.addfinalizer(lambda: teardown_auth(client))
 
     # test error
     with pytest.raises(ErrRootUserNotExist):
@@ -82,26 +85,29 @@ def test_auth_flow(client, request):
     assert len(r.roles) == 1
     assert r.roles[0] == 'root'
 
-    # test auth enable
-    client.auth_enable()
-    r = client.authenticate('root', 'root')
-    assert r.token
-
-    # test client auth
-    client.auth('root', 'root')
-    assert client.token
-    assert client.user_get('root')
-
-    # test user change password
-    client.user_change_password('root', 'changed')
-    client.auth('root', 'changed')
-    client.user_change_password('root', 'root')
-    client.auth('root', 'root')
-
-    # test auth disable
-    client.auth_disable()
-    with pytest.raises(ErrAuthNotEnabled):
-        client.authenticate('root', 'root')
+    if re.match(r'v3\.[0-2]\.{0,1}', ETCD_VER):
+        logging.info('skipping tests of apis with auth enabled, cause etcd < v3.3.0 does not support auth header')
+    else:
+        # test auth enable
+        client.auth_enable()
+        r = client.authenticate('root', 'root')
+        assert r.token
+
+        # test client auth
+        client.auth('root', 'root')
+        assert client.token
+        assert client.user_get('root')
+
+        # test user change password
+        client.user_change_password('root', 'changed')
+        client.auth('root', 'changed')
+        client.user_change_password('root', 'root')
+        client.auth('root', 'root')
+
+        # test auth disable
+        client.auth_disable()
+        with pytest.raises(ErrAuthNotEnabled):
+            client.authenticate('root', 'root')
 
     # test user revoke role
     client.user_revoke_role('root', 'root')

tests/test_py3/test_aio_auth_apis.py

@@ -1,9 +1,12 @@
 import asyncio
 
+import logging
 import pytest
+import re
 
 from etcd3 import AioClient
-from ..envs import protocol, host, port
+from tests.docker_cli import docker_run_etcd_main
+from ..envs import protocol, host, ETCD_VER
 from ..etcd_go_cli import NO_ETCD_SERVICE
 from ..etcd_go_cli import etcdctl
 
@@ -24,8 +27,8 @@ async def aio_client(event_loop, request):
     """
     init Etcd3Client, close its connection-pool when teardown
     """
-
-    c = AioClient(host, port, protocol)
+    _, p, _ = docker_run_etcd_main()
+    c = AioClient(host, p, protocol)
 
     def teardown():
         async def _t():
@@ -37,6 +40,7 @@ async def _t():
     request.addfinalizer(teardown)
     return c
 
+
 def teardown_auth():  # pragma: no cover
     """
     disable auth, delete all users and roles
@@ -55,14 +59,18 @@ def enable_auth():  # pragma: no cover
     etcdctl('user grant root root')
     etcdctl('auth enable')
 
+
 @pytest.mark.skipif(NO_ETCD_SERVICE, reason="no etcd service available")
 @pytest.mark.asyncio
 async def test_async_client_auth(aio_client, request):
+    teardown_auth()
     enable_auth()
     request.addfinalizer(teardown_auth)
 
     # test client auth
     await aio_client.auth('root', 'root')
     assert aio_client.token
-    assert aio_client.user_get('root')
-
+    if re.match(r'v3\.[0-2]\.{0,1}', ETCD_VER):
+        logging.info('skipping tests of apis with auth enabled, cause etcd < v3.3.0 does not support auth header')
+    else:
+        assert await aio_client.user_get('root')

tests/test_py3/test_aio_client.py

@@ -4,8 +4,9 @@
 import pytest
 
 from etcd3 import AioClient
-from ..docker_cli import docker_rm_etcd_ssl, docker_run_etcd_ssl, CERT_PATH, KEY_PATH, CA_PATH, NO_DOCKER_SERVICE
-from ..envs import protocol, host, port
+from ..docker_cli import docker_rm_etcd_ssl, docker_run_etcd_ssl, CERT_PATH, KEY_PATH, CA_PATH, NO_DOCKER_SERVICE, \
+    docker_run_etcd_main
+from ..envs import protocol, host
 from ..etcd_go_cli import NO_ETCD_SERVICE
 from ..etcd_go_cli import etcdctl
 
@@ -27,7 +28,8 @@ async def aio_client(event_loop, request):
     init Etcd3Client, close its connection-pool when teardown
     """
 
-    c = AioClient(host, port, protocol)
+    _, p, _ = docker_run_etcd_main()
+    c = AioClient(host, p, protocol)
 
     def teardown():
         async def _t():