From b63bec868f1b499ab981b40ebee6106d41e3c9bf Mon Sep 17 00:00:00 2001
From: Tony Aldon <tony@tonyaldon.com>
Date: Thu, 5 Oct 2023 12:00:45 +0200
Subject: [PATCH] pytest: clnrest test suite

- cln-grpc certificate reuse
- new certificate generation
- `GET` and `POST` requests
- websocket server
- config options and HTTP headers
---
 tests/test_clnrest.py | 435 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 435 insertions(+)
 create mode 100644 tests/test_clnrest.py

diff --git a/tests/test_clnrest.py b/tests/test_clnrest.py
new file mode 100644
index 000000000000..0be90caf7898
--- /dev/null
+++ b/tests/test_clnrest.py
@@ -0,0 +1,435 @@
+from ephemeral_port_reserve import reserve
+from fixtures import *  # noqa: F401,F403
+from pyln.testing.utils import env, wait_for, TEST_NETWORK
+import unittest
+import requests
+import shutil
+from pathlib import Path
+import socketio
+import time
+
+
+def test_clnrest_no_auto_start(node_factory):
+    """Ensure that we do not start clnrest unless a `rest-port` is configured."""
+    l1 = node_factory.get_node()
+    wait_for(lambda: [p for p in l1.rpc.plugin('list')['plugins'] if 'clnrest.py' in p['name']] == [])
+    assert l1.daemon.is_in_log(r'plugin-clnrest.py: Killing plugin: disabled itself at init: `rest-port` option is not configured')
+
+
+def test_clnrest_self_signed_certificates(node_factory):
+    """Test that self-signed certificates have `rest-host` IP in Subject Alternative Name."""
+    rest_port = str(reserve())
+    rest_host = '127.0.0.1'
+    base_url = f'https://{rest_host}:{rest_port}'
+    l1 = node_factory.get_node(options={'disable-plugin': 'cln-grpc',
+                                        'rest-port': rest_port,
+                                        'rest-host': rest_host})
+    wait_for(lambda: l1.daemon.is_in_log(r'plugin-clnrest.py: REST server running at ' + base_url))
+    ca_cert = Path(l1.daemon.lightning_dir) / TEST_NETWORK / 'ca.pem'
+    response = requests.get(base_url + '/v1/list-methods', verify=ca_cert)
+    assert response.status_code == 200
+
+
+@unittest.skipIf(env('RUST') != '1', 'RUST is not enabled skipping rust-dependent tests')
+def test_clnrest_uses_grpc_plugin_certificates(node_factory):
+    """Test that clnrest reuses `cln-grpc` plugin certificates if available.
+    Defaults:
+    - rest-protocol: https
+    """
+    rest_host = 'localhost'
+    grpc_port = str(reserve())
+    rest_port = str(reserve())
+    l1 = node_factory.get_node(options={'grpc-port': grpc_port, 'rest-host': rest_host, 'rest-port': rest_port})
+    base_url = f'https://{rest_host}:{rest_port}'
+    wait_for(lambda: l1.daemon.is_in_log(r'serving grpc on 0.0.0.0:'))
+    wait_for(lambda: l1.daemon.is_in_log(r'plugin-clnrest.py: REST server running at ' + base_url))
+    ca_cert = Path(l1.daemon.lightning_dir) / TEST_NETWORK / 'ca.pem'
+    response = requests.get(base_url + '/v1/list-methods', verify=ca_cert)
+    assert response.status_code == 200
+
+
+def test_clnrest_generate_certificate(node_factory):
+    """Test whether we correctly generate the certificates."""
+    # when `rest-protocol` is `http`, certs are not generated at `rest-certs` path
+    rest_port = str(reserve())
+    rest_protocol = 'http'
+    rest_certs = '/tmp/ltests-certs'
+    # delete existing certs and not generate new with `http` option
+    shutil.rmtree(Path(rest_certs), ignore_errors=True)
+    l1 = node_factory.get_node(options={'rest-port': rest_port,
+                                        'rest-protocol': rest_protocol,
+                                        'rest-certs': rest_certs})
+
+    assert not Path(rest_certs).exists()
+
+    # node l1 not started
+    rest_port = str(reserve())
+    rest_certs = '/tmp/ltests-certs'
+    # delete existing certs and generate new with updated host options
+    shutil.rmtree(Path(rest_certs), ignore_errors=True)
+    l1 = node_factory.get_node(options={'rest-port': rest_port,
+                                        'rest-certs': rest_certs}, start=False)
+    rest_certs_path = Path(rest_certs)
+    files = [rest_certs_path / f for f in [
+        'ca.pem',
+        'ca-key.pem',
+        'client.pem',
+        'client-key.pem',
+        'server-key.pem',
+        'server.pem',
+    ]]
+
+    # before starting no files exist.
+    assert [f.exists() for f in files] == [False] * len(files)
+
+    # certificates generated at startup
+    l1.start()
+    assert [f.exists() for f in files] == [True] * len(files)
+
+    # the files exist, restarting should not change them
+    contents = [f.open().read() for f in files]
+    l1.restart()
+    assert contents == [f.open().read() for f in files]
+
+    # # remove client.pem file, so all certs are regenerated at restart
+    files[2].unlink()
+    l1.restart()
+    contents_1 = [f.open().read() for f in files]
+    assert [c[0] != c[1] for c in zip(contents, contents_1)] == [True] * len(files)
+
+    # remove client-key.pem file, so all certs are regenerated at restart
+    files[3].unlink()
+    l1.restart()
+    contents_2 = [f.open().read() for f in files]
+    assert [c[0] != c[1] for c in zip(contents, contents_2)] == [True] * len(files)
+
+
+def start_node_with_clnrest(node_factory):
+    """Start a node with the clnrest plugin, whose options are the default options.
+    Return:
+    - the node,
+    - the base url and
+    - the certificate authority path used for the self-signed certificates."""
+    rest_port = str(reserve())
+    rest_certs = '/tmp/ltests-certs'
+    # delete existing certs and generate new with updated host options
+    shutil.rmtree(Path(rest_certs), ignore_errors=True)
+    l1 = node_factory.get_node(options={'rest-port': rest_port, 'rest-certs': rest_certs})
+    base_url = 'https://127.0.0.1:' + rest_port
+    wait_for(lambda: l1.daemon.is_in_log(r'plugin-clnrest.py: REST server running at ' + base_url))
+    ca_cert = Path(rest_certs) / 'ca.pem'
+    return l1, base_url, ca_cert
+
+
+def test_clnrest_list_methods(node_factory):
+    """Test GET request on `/v1/list-methods` end point with default values for options."""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+
+    # /v1/list-methods
+    response = requests.get(base_url + '/v1/list-methods', verify=ca_cert)
+    assert response.status_code == 200
+    assert response.text.find('Command: getinfo') > 0
+
+
+def test_clnrest_unknown_method(node_factory):
+    """Test GET request error on `/v1/unknown-get` end point with default values for options."""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+
+    response = requests.get(base_url + '/v1/unknown-get', verify=ca_cert)
+    assert response.status_code == 405
+    assert response.json()['message'] == 'The method is not allowed for the requested URL.'
+
+    """Test POST request error on `/v1/unknown-post` end point."""
+    rune = l1.rpc.createrune()['rune']
+    response = requests.post(base_url + '/v1/unknown-post', headers={'Rune': rune}, verify=ca_cert)
+    assert response.status_code == 500
+    assert response.json()['error']['code'] == -32601
+    assert response.json()['error']['message'] == "Unknown command 'unknown-post'"
+
+
+def test_clnrest_rpc_method(node_factory):
+    """Test POST requests on `/v1/<rpc_method>` end points with default values for options."""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+
+    # /v1/getinfo no rune provided in header of the request
+    response = requests.post(base_url + '/v1/getinfo', verify=ca_cert)
+    assert response.status_code == 401
+    assert response.json()['error']['code'] == 403
+    assert response.json()['error']['message'] == 'Not authorized: Missing rune'
+
+    # /v1/getinfo with a rune which doesn't authorized getinfo method
+    rune_no_getinfo = l1.rpc.createrune(restrictions=[["method/getinfo"]])['rune']
+    response = requests.post(base_url + '/v1/getinfo', headers={'Rune': rune_no_getinfo},
+                             verify=ca_cert)
+    assert response.status_code == 401
+    assert response.json()['error']['code'] == 1502
+    assert response.json()['error']['message'] == 'Not permitted: method is equal to getinfo'
+
+    # /v1/getinfo with a correct rune
+    rune_getinfo = l1.rpc.createrune(restrictions=[["method=getinfo"]])['rune']
+    response = requests.post(base_url + '/v1/getinfo', headers={'Rune': rune_getinfo},
+                             verify=ca_cert)
+    assert response.status_code == 201
+    assert response.json()['id'] == l1.info['id']
+
+    # /v1/invoice with a correct rune but missing parameters
+    rune_invoice = l1.rpc.createrune(restrictions=[["method=invoice"]])['rune']
+    response = requests.post(base_url + '/v1/invoice', headers={'Rune': rune_invoice},
+                             verify=ca_cert)
+    assert response.status_code == 500
+    assert response.json()['error']['code'] == -32602
+
+    # /v1/invoice with a correct rune but wrong parameters
+    rune_invoice = l1.rpc.createrune(restrictions=[["method=invoice"]])['rune']
+    response = requests.post(base_url + '/v1/invoice', headers={'Rune': rune_invoice},
+                             data={'amount_msat': '<WRONG>',
+                                   'label': 'label',
+                                   'description': 'description'},
+                             verify=ca_cert)
+    assert response.status_code == 500
+    assert response.json()['error']['code'] == -32602
+
+    # l2 pays l1's invoice where the invoice is created with /v1/invoice
+    rune_invoice = l1.rpc.createrune(restrictions=[["method=invoice"]])['rune']
+    response = requests.post(base_url + '/v1/invoice', headers={'Rune': rune_invoice},
+                             data={'amount_msat': '50000000',
+                                   'label': 'label',
+                                   'description': 'description'},
+                             verify=ca_cert)
+    assert response.status_code == 201
+    assert 'bolt11' in response.json()
+
+
+# Tests for websocket are written separately to avoid flake8
+# to complain with the errors F811 like this "F811 redefinition of
+# unused 'message'".
+
+def notifications_received_via_websocket(l1, base_url, http_session):
+    """Return the list of notifications received by the websocket client.
+
+    We try to connect to the websocket server running at `base_url`
+    with `http_session` parameters.  Then we create an invoice on the node:
+
+    - if we were effectively connected, we received an `invoice_creation`
+      notification via websocket that should be in the list of notifications
+      we return.
+    - if we couldn't connect to the websocket server, the notification list
+      we return is empty."""
+    sio = socketio.Client(http_session=http_session)
+    notifications = []
+
+    @sio.event
+    def message(data):
+        notifications.append(data)
+    sio.connect(base_url)
+    sio.sleep(2)
+    # trigger `invoice_creation` notification
+    l1.rpc.invoice(10000, "label", "description")
+    time.sleep(2)
+    sio.disconnect()
+    return notifications
+
+
+def test_clnrest_websocket_no_rune(node_factory):
+    """Test websocket with default values for options."""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+
+    # http session
+    http_session = requests.Session()
+    http_session.verify = ca_cert.as_posix()
+
+    # no rune provided => no websocket connection and no notification received
+    notifications = notifications_received_via_websocket(l1, base_url, http_session)
+    assert len(notifications) == 0
+
+
+def test_clnrest_websocket_wrong_rune(node_factory):
+    """Test websocket with default values for options."""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+
+    # http session
+    http_session = requests.Session()
+    http_session.verify = ca_cert.as_posix()
+
+    # wrong rune provided => no websocket connection and no notification received
+    http_session.headers.update({"rune": "<WRONG>"})
+    notifications = notifications_received_via_websocket(l1, base_url, http_session)
+    assert len(notifications) == 0
+
+
+def test_clnrest_websocket_unrestricted_rune(node_factory):
+    """Test websocket with default values for options."""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+
+    # http session
+    http_session = requests.Session()
+    http_session.verify = ca_cert.as_posix()
+
+    # unrestricted rune provided => websocket connection and notifications received
+    rune_unrestricted = l1.rpc.createrune()['rune']
+    http_session.headers.update({"rune": rune_unrestricted})
+    notifications = notifications_received_via_websocket(l1, base_url, http_session)
+    assert len([n for n in notifications if n.find('invoice_creation') > 0]) == 1
+
+
+def test_clnrest_websocket_rune_readonly(node_factory):
+    """Test websocket with default values for options."""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+
+    # http session
+    http_session = requests.Session()
+    http_session.verify = ca_cert.as_posix()
+
+    # readonly rune provided => websocket connection and notifications received
+    rune_readonly = l1.rpc.createrune(restrictions="readonly")['rune']
+    http_session.headers.update({"rune": rune_readonly})
+    notifications = notifications_received_via_websocket(l1, base_url, http_session)
+    assert len([n for n in notifications if n.find('invoice_creation') > 0]) == 1
+
+
+def test_clnrest_websocket_rune_listnotifications(node_factory):
+    """Test websocket with default values for options."""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+
+    # http session
+    http_session = requests.Session()
+    http_session.verify = ca_cert.as_posix()
+
+    # rune authorizing listclnrest-notifications method provided => websocket connection and notifications received
+    rune_clnrest_notifications = l1.rpc.createrune(restrictions=[["method=listclnrest-notifications"]])['rune']
+    http_session.headers.update({"rune": rune_clnrest_notifications})
+    notifications = notifications_received_via_websocket(l1, base_url, http_session)
+    assert len([n for n in notifications if n.find('invoice_creation') > 0]) == 1
+
+
+def test_clnrest_websocket_rune_no_listnotifications(node_factory):
+    """Test websocket with default values for options."""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+
+    # http session
+    http_session = requests.Session()
+    http_session.verify = ca_cert.as_posix()
+
+    # with a rune which doesn't authorized listclnrest-notifications method => no websocket connection and no notification received
+    rune_no_clnrest_notifications = l1.rpc.createrune(restrictions=[["method/listclnrest-notifications"]])['rune']
+    http_session.headers.update({"rune": rune_no_clnrest_notifications})
+    notifications = notifications_received_via_websocket(l1, base_url, http_session)
+    assert len([n for n in notifications if n.find('invoice_creation') > 0]) == 0
+
+
+def test_clnrest_options(node_factory):
+    """Test startup options `rest-host`, `rest-protocol` and `rest-certs`."""
+    # with invalid port
+    rest_port = 1000
+    l1 = node_factory.get_node(options={'rest-port': rest_port})
+    base_url = 'https://127.0.0.1:1000'
+    assert l1.daemon.is_in_log(r'plugin-clnrest.py: Killing plugin: disabled itself at init: `rest-port` should be a valid available port between 1024 and 65535')
+    assert not l1.daemon.is_in_log(r'plugin-clnrest.py: REST server running at ' + base_url)
+
+    # with invalid protocol
+    rest_port = str(reserve())
+    rest_protocol = 'htttps'
+    l1 = node_factory.get_node(options={'rest-port': rest_port,
+                                        'rest-protocol': rest_protocol})
+    base_url = 'https://127.0.0.1:' + rest_port
+    assert l1.daemon.is_in_log(r'plugin-clnrest.py: `rest-protocol` can either be http or https. Resetting to default `https`')
+    assert l1.daemon.is_in_log(r'plugin-clnrest.py: REST server running at ' + base_url)
+
+    # with invalid host
+    rest_port = str(reserve())
+    rest_host = '127.0.0.12.15'
+    l1 = node_factory.get_node(options={'rest-port': rest_port,
+                                        'rest-host': rest_host})
+    base_url = 'https://127.0.0.1:' + rest_port
+    assert l1.daemon.is_in_log(r'plugin-clnrest.py: `rest-host` should be a valid IP. Resetting to default `127.0.0.1`')
+    assert l1.daemon.is_in_log(r'plugin-clnrest.py: REST server running at ' + base_url)
+
+    # with `http` protocol and custom host
+    rest_port = str(reserve())
+    rest_host = '127.0.0.2'
+    rest_protocol = 'http'
+    l1 = node_factory.get_node(options={'rest-port': rest_port,
+                                        'rest-host': rest_host,
+                                        'rest-protocol': rest_protocol})
+    base_url = rest_protocol + '://' + rest_host + ':' + rest_port
+    wait_for(lambda: l1.daemon.is_in_log(r'plugin-clnrest.py: REST server running at ' + base_url))
+
+    # /v1/list-methods
+    response = requests.get(base_url + '/v1/list-methods')
+    assert response.status_code == 200
+
+    # /v1/getinfo
+    rune_getinfo = l1.rpc.createrune(restrictions=[["method=getinfo"]])['rune']
+    response = requests.post(base_url + '/v1/getinfo', headers={'Rune': rune_getinfo})
+    assert response.status_code == 201
+
+    # start node l1 with clnrest listening at `base_url` with certificate `ca_cert`
+    rest_port = str(reserve())
+    rest_host = '127.0.0.2'
+    rest_certs = '/tmp/ltests-certs'
+    # delete existing certs and generate new with updated host options
+    shutil.rmtree(Path(rest_certs), ignore_errors=True)
+    l1 = node_factory.get_node(options={'rest-port': rest_port,
+                                        'rest-host': rest_host,
+                                        'rest-certs': rest_certs})
+    base_url = 'https://' + rest_host + ':' + rest_port
+    wait_for(lambda: l1.daemon.is_in_log(r'plugin-clnrest.py: REST server running at ' + base_url))
+
+    rest_certs_default = Path(l1.daemon.lightning_dir) / TEST_NETWORK
+    ca_cert = Path(rest_certs) / 'ca.pem'
+    assert rest_certs_default != ca_cert
+
+    # /v1/list-methods
+    response = requests.get(base_url + '/v1/list-methods', verify=ca_cert)
+    assert response.status_code == 200
+
+    # /v1/getinfo
+    rune_getinfo = l1.rpc.createrune(restrictions=[["method=getinfo"]])['rune']
+    response = requests.post(base_url + '/v1/getinfo', headers={'Rune': rune_getinfo},
+                             verify=ca_cert)
+    assert response.status_code == 201
+
+
+def test_clnrest_http_headers(node_factory):
+    """Test HTTP headers set with `rest-csp` and `rest-cors-origins` options."""
+    # start a node with clnrest
+    l1, base_url, ca_cert = start_node_with_clnrest(node_factory)
+
+    # Default values for `rest-csp` and `rest-cors-origins` options
+    response = requests.get(base_url + '/v1/list-methods', verify=ca_cert)
+    assert response.headers['Content-Security-Policy'] == "default-src 'self'; font-src 'self'; img-src 'self' data:; frame-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline';"
+    assert response.headers['Access-Control-Allow-Origin'] == '*'
+
+    # Custom values for `rest-csp` and `rest-cors-origins` options
+    rest_port = str(reserve())
+    rest_certs = '/tmp/ltests-certs'
+    # delete existing certs and generate new with updated host options
+    shutil.rmtree(Path(rest_certs), ignore_errors=True)
+    l2 = node_factory.get_node(options={
+        'rest-port': rest_port,
+        'rest-certs': rest_certs,
+        'rest-csp': "default-src 'self'; font-src 'self'; img-src 'self'; frame-src 'self'; style-src 'self'; script-src 'self';",
+        'rest-cors-origins': ['https://localhost:5500', 'http://192.168.1.30:3030', 'http://192.168.1.10:1010']
+    })
+    base_url = 'https://127.0.0.1:' + rest_port
+    wait_for(lambda: l2.daemon.is_in_log(r'plugin-clnrest.py: REST server running at ' + base_url))
+    ca_cert = Path(rest_certs) / 'ca.pem'
+
+    response = requests.get(base_url + '/v1/list-methods',
+                            headers={'Origin': 'http://192.168.1.30:3030'},
+                            verify=ca_cert)
+    assert response.headers['Content-Security-Policy'] == "default-src 'self'; font-src 'self'; img-src 'self'; frame-src 'self'; style-src 'self'; script-src 'self';"
+    assert response.headers['Access-Control-Allow-Origin'] == 'http://192.168.1.30:3030'
+    response = requests.get(base_url + '/v1/list-methods',
+                            headers={'Origin': 'http://192.168.1.10:1010'},
+                            verify=ca_cert)
+    assert response.headers['Access-Control-Allow-Origin'] == 'http://192.168.1.10:1010'