Merge remote-tracking branch 'upstream/master'

.gitattributes

@@ -17,4 +17,4 @@ statements_gettextgen.po linguist-generated=true
 cln-grpc/proto/node.proto -text -diff linguist-generated=true
 cln-grpc/src/convert.rs -text -diff linguist-generated=true
 cln-rpc/src/model.rs -text -diff linguist-generated=true
-contrib/pyln-testing/pyln/testing/node_pb2.py linguist-generated=true
+contrib/pyln-testing/pyln/testing/node_pb2.py -text -diff linguist-generated=true

Cargo.toml

@@ -2,6 +2,7 @@
 strip = "debuginfo"
 
 [workspace]
+resolver = "2"
 members = [
   "cln-rpc",
   "cln-grpc",

Makefile

@@ -368,7 +368,7 @@ ifneq ($(RUST),0)
 	include cln-grpc/Makefile
 
 $(MSGGEN_GENALL)&: doc/schemas/*.request.json doc/schemas/*.schema.json
-	PYTHONPATH=contrib/msggen python3 contrib/msggen/msggen/__main__.py
+	PYTHONPATH=contrib/msggen $(PYTHON) contrib/msggen/msggen/__main__.py
 
 # The compiler assumes that the proto files are in the same
 # directory structure as the generated files will be. Since we
@@ -384,8 +384,8 @@ GRPC_GEN = \
 ALL_TEST_GEN += $(GRPC_GEN)
 
 $(GRPC_GEN)&: cln-grpc/proto/node.proto cln-grpc/proto/primitives.proto
-	python3 -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/node.proto --python_out=$(GRPC_PATH)/ --grpc_python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
-	python3 -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/primitives.proto --python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
+	$(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/node.proto --python_out=$(GRPC_PATH)/ --grpc_python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
+	$(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/primitives.proto --python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
 	find $(GRPC_DIR)/ -type f -name "*.py" -print0 | xargs -0 sed -i'.bak' -e 's/^import \(.*\)_pb2 as .*__pb2/from pyln.grpc import \1_pb2 as \1__pb2/g'
 	find $(GRPC_DIR)/ -type f -name "*.py.bak" -delete
 endif
@@ -421,7 +421,7 @@ mkdocs.yml: $(MANPAGES:=.md)
 	  find doc -maxdepth 1 -name '*\.[0-9]\.md' | \
 	  cut -b 5- | LC_ALL=C sort | \
 	  sed 's/\(.*\)\.\(.*\).*\.md/- "\1": "\1.\2.md"/' | \
-	  python3 devtools/blockreplace.py mkdocs.yml manpages --language=yml --indent "          " \
+	  $(PYTHON) devtools/blockreplace.py mkdocs.yml manpages --language=yml --indent "          " \
 	)
 
 

cln-grpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cln-grpc"
-version = "0.1.3"
+version = "0.1.4"
 edition = "2021"
 license = "MIT"
 description = "The Core Lightning API as grpc primitives. Provides the bindings used to expose the API over the network."

cln-rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cln-rpc"
-version = "0.1.3"
+version = "0.1.4"
 edition = "2021"
 license = "MIT"
 description = "An async RPC client for Core Lightning."

configure

@@ -69,11 +69,25 @@ default_cwarnflags()
     echo "$F"
 }
 
+default_python()
+{
+    PYTHON_BINS="python3 python"
+    for p in $PYTHON_BINS; do
+        if [ "$(which $p)" != "" ] ; then
+            if $p --version 2>&1 | grep -q "Python 3."; then
+                echo "$p"
+                return
+            fi
+        fi
+    done
+}
+
+# Takes PYTHON var
 default_pytest()
 {
     # Since we just checked that we have python3 we give that one the
     # most priority and then fall back to some common aliases.
-    PYTEST_BINS="python3 -m pytest,pytest,py.test,pytest3,pytest-3"
+    PYTEST_BINS="$1 -m pytest,pytest,py.test,pytest3,pytest-3"
     IFS=','
     for p in $PYTEST_BINS; do
 	# If it is a combined command such as `python3 -m pytest` we
@@ -87,16 +101,10 @@ default_pytest()
 	fi
     done
 
-    PYTHON_BINS="python python3"
-    for p in $PYTHON_BINS; do
-        if [ "$(which $p)" != "" ] ; then
-            $p --version 2>&1 | grep -q "Python 3." || continue
-            if $p -c "import pytest" 2>/dev/null ; then
-                echo "$p -m pytest"
-                return
-            fi
-        fi
-    done
+    if $1 -c "import pytest" 2>/dev/null; then
+        echo "$1 -m pytest"
+        return
+    fi
 }
 
 check_command()
@@ -170,7 +178,8 @@ set_defaults()
 	CSANFLAGS="$CSANFLAGS $FUZZFLAGS"
     fi
     echo CSANFLAGS = $CSANFLAGS
-    PYTEST=${PYTEST-$(default_pytest)}
+    PYTHON=${PYTHON-$(default_python)}
+    PYTEST=${PYTEST-$(default_pytest $PYTHON)}
     COPTFLAGS=${COPTFLAGS-$(default_coptflags "$DEBUGBUILD")}
     CONFIGURATOR_CC=${CONFIGURATOR_CC-$CC}
     VALGRIND=${VALGRIND:-$(default_valgrind_setting)}
@@ -526,6 +535,7 @@ add_var HAVE_LOWDOWN "$HAVE_LOWDOWN"
 add_var SHA256SUM "$SHA256SUM"
 add_var FUZZING "$FUZZING"
 add_var RUST "$RUST"
+add_var PYTHON "$PYTHON"
 
 # Hack to avoid sha256 name clash with libwally: will be fixed when that
 # becomes a standalone shared lib.

doc/Makefile

@@ -235,7 +235,7 @@ doc/index.rst: $(MANPAGES:=.md)
 	  find doc -maxdepth 1 -name '*\.[0-9]\.md' | \
 	  cut -b 5- | LC_ALL=C sort | \
 	  sed "s/\(.*\)\.\(.*\).*\.md/\1 <\1.\2.md>/" | \
-	  python3 devtools/blockreplace.py doc/index.rst manpages --language=rst --indent "   " \
+	  $(PYTHON) devtools/blockreplace.py doc/index.rst manpages --language=rst --indent "   " \
 	)
 
 # Overridden by GH CI if necessary.

doc/getting-started/getting-started/installation.md

@@ -17,7 +17,11 @@ sudo snap install groestlcoin-core
 sudo ln -s /snap/groestlcoin-core/current/bin/groestlcoin{d,-cli} /usr/local/bin/
 ```
 
-Then you can fetch a pre-compiled binary from the [releases](https://github.com/Groestlcoin/lightning/releases) page on GitHub. Core Lightning provides binaries for both Ubuntu and Fedora distributions.
+Then you can fetch a pre-compiled binary from the [releases](https://github.com/Groestlcoin/lightning/releases) page on GitHub. Core Lightning provides binaries for both Ubuntu and Fedora distributions. Normally these binaries are extracted into /usr/local:
+
+```shell
+sudo tar -xvf <release>.tar.xz -C /usr/local --strip-components=2
+```
 
 You will need some Python packages if you want to use clnrest.  Unfortunately there are some Python packages which are not packaged in Ubuntu, and so you will need to force installation of these (I recommend --user which will install them in your own .local directory, so at least you won't run the risk of breaking Python globally!).
 

doc/schemas/getroute.request.json

@@ -20,7 +20,7 @@
       "description": ""
     },
     "cltv": {
-      "type": "number",
+      "type": "u32",
       "description": ""
     },
     "fromid": {

plugins/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cln-plugin"
-version = "0.1.4"
+version = "0.1.5"
 edition = "2021"
 license = "MIT"
 description = "A CLN plugin library. Write your plugin in Rust."

plugins/clnrest/utilities/rpc_routes.py

@@ -48,7 +48,10 @@ def post(self, rpc_method):
 
         try:
             if request.is_json:
-                payload = request.get_json()
+                if len(request.data) != 0:
+                    payload = request.get_json()
+                else:
+                    payload = {}
             else:
                 payload = request.form.to_dict()
             return call_rpc_method(plugin, rpc_method, payload), 201

plugins/clnrest/utilities/shared.py

@@ -53,7 +53,10 @@ def verify_rune(plugin, request):
         raise Exception('{ "error": {"code": 403, "message": "Not authorized: Missing rune"} }')
 
     if request.is_json:
-        rpc_params = request.get_json()
+        if len(request.data) != 0:
+            rpc_params = request.get_json()
+        else:
+            rpc_params = {}
     else:
         rpc_params = request.form.to_dict()
 

plugins/grpc-plugin/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 edition = "2021"
 name = "cln-grpc-plugin"
-version = "0.1.2"
+version = "0.1.4"
 
 [[bin]]
 name = "cln-grpc"

plugins/renepay/pay.c

@@ -295,11 +295,11 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 		      "{code:%,message:%}",
 		      JSON_SCAN(json_to_jsonrpc_errcode, &errcode),
 		      JSON_SCAN_TAL(tmpctx, json_strdup, &msg))) {
-		plugin_err(cmd->plugin, "Bad fail from sendpay: %.*s",
+		plugin_err(pay_plugin->plugin, "Bad fail from sendpay: %.*s",
 			   json_tok_full_len(err), json_tok_full(buf, err));
 	}
 	if (errcode != PAY_TRY_OTHER_ROUTE)
-		plugin_err(cmd->plugin, "Strange error from sendpay: %.*s",
+		plugin_err(pay_plugin->plugin, "Strange error from sendpay: %.*s",
 			   json_tok_full_len(err), json_tok_full(buf, err));
 
 	/* There is no new knowledge from this kind of failure.
@@ -350,7 +350,19 @@ static void sendpay_new_flows(struct payment *p)
 		json_add_sha256(req->js, "payment_hash", &p->payment_hash);
 		json_add_secret(req->js, "payment_secret", p->payment_secret);
 
-		json_add_amount_msat(req->js, "amount_msat", p->amount);
+		/* FIXME: sendpay has a check that we don't total more than
+		 * the exact amount, if we're setting partid (i.e. MPP).  However,
+		 * we always set partid, and we add a shadow amount *if we've
+		 * only have one part*, so we have to use that amount here.
+		 *
+		 * The spec was loosened so you are actually allowed
+		 * to overpay, so this check is now overzealous. */
+		if (amount_msat_greater(payflow_delivered(pf), p->amount)) {
+			json_add_amount_msat(req->js, "amount_msat",
+					     payflow_delivered(pf));
+		} else {
+			json_add_amount_msat(req->js, "amount_msat", p->amount);
+		}
 
 		json_add_u64(req->js, "partid", pf->key.partid);
 

plugins/renepay/pay_flow.c

@@ -115,7 +115,7 @@ static u32 shadow_one_flow(const struct gossmap *gossmap,
 	/* We only create shadow for extra CLTV delays, *not* for
 	 * amounts.  This is because with MPP our amounts are random
 	 * looking already. */
-	for (hop = 0; hop < MAX_SHADOW_LEN && pseudorand(1); hop++) {
+	for (hop = 0; hop < MAX_SHADOW_LEN && pseudorand(2); hop++) {
 		/* Try for a believable channel up to 10 times, then stop */
 		for (size_t i = 0; i < 10; i++) {
 			struct amount_sat cap;
@@ -124,7 +124,7 @@ static u32 shadow_one_flow(const struct gossmap *gossmap,
 			if (!gossmap_chan_set(chans[hop], dirs[hop])
 			    || !gossmap_chan_get_capacity(gossmap, chans[hop], &cap)
 			    /* This test is approximate, since amount would differ */
-			    || amount_msat_less_sat(amount, cap)) {
+			    || amount_msat_greater_sat(amount, cap)) {
 				chans[hop] = NULL;
 				continue;
 			}
@@ -172,8 +172,8 @@ static bool add_to_amounts(const struct gossmap *gossmap,
 	for (int i = num-2; i >= 0; i--) {
 		amounts[i] = amounts[i+1];
 		if (!amount_msat_add_fee(&amounts[i],
-					 flow_edge(f, i)->base_fee,
-					 flow_edge(f, i)->proportional_fee))
+					 flow_edge(f, i+1)->base_fee,
+					 flow_edge(f, i+1)->proportional_fee))
 			return false;
 	}
 
@@ -322,7 +322,7 @@ static void convert_and_attach_flows(struct payment *payment,
 		pf->cltv_delays[plen-1] = final_cltvs[i];
 		for (int j = (int)plen-2; j >= 0; j--) {
 			pf->cltv_delays[j] = pf->cltv_delays[j+1]
-				+ f->path[j]->half[f->dirs[j]].delay;
+				+ f->path[j+1]->half[f->dirs[j+1]].delay;
 		}
 		pf->amounts = tal_steal(pf, f->amounts);
 		pf->success_prob = f->success_prob;

tests/test_renepay.py

@@ -18,6 +18,42 @@ def test_simple(node_factory):
     assert details['destination'] == l2.info['id']
 
 
+def test_direction_matters(node_factory):
+    '''Make sure we use correct delay and fees for the direction we're going.'''
+    l1, l2, l3 = node_factory.line_graph(3,
+                                         wait_for_announce=True,
+                                         opts=[{},
+                                               {'fee-base': 2000, 'fee-per-satoshi': 20, 'cltv-delta': 20},
+                                               {'fee-base': 3000, 'fee-per-satoshi': 30, 'cltv-delta': 30}])
+    inv = l3.rpc.invoice(123000, 'test_renepay', 'description')['bolt11']
+    details = l1.rpc.call('renepay', {'invstring': inv})
+    assert details['status'] == 'complete'
+    assert details['amount_msat'] == Millisatoshi(123000)
+    assert details['destination'] == l3.info['id']
+
+
+def test_shadow(node_factory):
+    '''Make sure we shadow correctly.'''
+    l1, l2, l3 = node_factory.line_graph(3,
+                                         wait_for_announce=True,
+                                         opts=[{},
+                                               {'fee-base': 2000, 'fee-per-satoshi': 20, 'cltv-delta': 20},
+                                               {'fee-base': 3000, 'fee-per-satoshi': 30, 'cltv-delta': 30}])
+
+    # Shadow doesn't always happen (50% chance)!
+    for i in range(20):
+        inv = l2.rpc.invoice(123000, f'test_renepay{i}', 'description')['bolt11']
+        details = l1.rpc.call('renepay', {'invstring': inv})
+        assert details['status'] == 'complete'
+        assert details['amount_msat'] == Millisatoshi(123000)
+        assert details['destination'] == l2.info['id']
+
+        line = l1.daemon.wait_for_log("No MPP, so added .*msat shadow fee")
+        if 'added 0msat' not in line:
+            break
+        assert i != 19
+
+
 def test_mpp(node_factory):
     '''Test paying a remote node using two routes.
     1----2----4