Merge branch 'master' into sql

ElementsProject · Jul 12, 2024 · 797cdc8 · 797cdc8
2 parents 5b687dd + 05f5976
commit 797cdc8
Show file tree

Hide file tree

Showing 357 changed files with 11,707 additions and 11,173 deletions.
diff --git a/.github/scripts/install-bitcoind.sh b/.github/scripts/install-bitcoind.sh
@@ -2,6 +2,9 @@
 
 set -e
 
+export BITCOIN_VERSION=27.1
+export ELEMENTS_VERSION=23.2.1
+
 DIRNAME="bitcoin-${BITCOIN_VERSION}"
 EDIRNAME="elements-${ELEMENTS_VERSION}"
 FILENAME="${DIRNAME}-x86_64-linux-gnu.tar.gz"

diff --git a/.github/scripts/setup.sh b/.github/scripts/setup.sh
@@ -1,8 +1,6 @@
 #!/bin/bash
 set -e
 export DEBIAN_FRONTEND=noninteractive
-export BITCOIN_VERSION=26.1
-export ELEMENTS_VERSION=22.0.2
 export RUST_VERSION=stable
 
 sudo useradd -ms /bin/bash tester
@@ -53,20 +51,7 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
 echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
 sudo chmod 0440 /etc/sudoers.d/tester
 
-(
-    cd /tmp/ || exit 1
-    wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
-    wget https://github.com/ElementsProject/elements/releases/download/elements-${ELEMENTS_VERSION}/elements-${ELEMENTS_VERSION}-x86_64-linux-gnu.tar.gz
-    tar -xf bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
-    tar -xf elements-${ELEMENTS_VERSION}-x86_64-linux-gnu.tar.gz
-    sudo mv bitcoin-${BITCOIN_VERSION}/bin/* /usr/local/bin
-    sudo mv elements-${ELEMENTS_VERSION}/bin/* /usr/local/bin
-    rm -rf \
-       bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz \
-       bitcoin-${BITCOIN_VERSION} \
-       elements-${ELEMENTS_VERSION}-x86_64-linux-gnu.tar.gz \
-       elements-${ELEMENTS_VERSION}
-)
+"$(dirname "$0")"/install-bitcoind.sh
 
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
      -y --default-toolchain ${RUST_VERSION}

diff --git a/.github/scripts/sync-rpc-cmds.py b/.github/scripts/sync-rpc-cmds.py
@@ -2,26 +2,30 @@
 from time import sleep
 import requests
 import re
+from enum import Enum
 
 # readme url
-URL = "https://dash.readme.com/api/v1/docs"
+URL = "https://dash.readme.com/api/v1"
 # category id for API reference
 CATEGORY_ID = "63e4e160c60b2e001dd1cc4e"
+CATEGORY_SLUG = "json-rpc-apis"
 
 
-def checkIfDocIsPresent(title, headers):
+class Action(Enum):
+    ADD = 'add'
+    UPDATE = 'update'
+    DELETE = 'delete'
 
-    check_url = URL + "/" + title
-    response = requests.get(check_url, headers=headers)
 
+def getListOfRPCDocs(headers):
+    response = requests.get(f"{URL}/categories/{CATEGORY_SLUG}/docs", headers=headers)
     if response.status_code == 200:
-        return True
+        return response.json()
     else:
-        return False
+        return []
 
 
-def publishDoc(title, body, order):
-    key = os.environ.get("README_API_KEY")
+def publishDoc(action, title, body, order, headers):
     payload = {
         "title": title,
         "type": "basic",
@@ -30,28 +34,30 @@ def publishDoc(title, body, order):
         "hidden": False,
         "order": order,
     }
-    headers = {
-        "accept": "application/json",
-        "content-type": "application/json",
-        "authorization": "Basic " + key,
-    }
-
-    isDocPresent = checkIfDocIsPresent(title, headers)
-    if isDocPresent:
+    # title == slug
+    if action == Action.ADD:
+        # create doc
+        response = requests.post(URL + "/docs", json=payload, headers=headers)
+        if response.status_code != 201:
+            print(response.text)
+        else:
+            print("Created ", title)
+    elif action == Action.UPDATE:
         # update doc
-        update_url = URL + "/" + title  # title == slug
-        response = requests.put(update_url, json=payload, headers=headers)
+        response = requests.put(f"{URL}/docs/{title}", json=payload, headers=headers)
         if response.status_code != 200:
             print(response.text)
         else:
             print("Updated ", title)
-    else:
-        # create doc
-        response = requests.post(URL, json=payload, headers=headers)
-        if response.status_code != 201:
+    elif action == Action.DELETE:
+        # delete doc
+        response = requests.delete(f"{URL}/docs/{title}", headers=headers)
+        if response.status_code != 204:
             print(response.text)
         else:
-            print("Created ", title)
+            print("Deleted ", title)
+    else:
+        print("Invalid action")
 
 
 def extract_rpc_commands(rst_content):
@@ -69,19 +75,37 @@ def extract_rpc_commands(rst_content):
 
 
 def main():
+    # define headers for requests
+    headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+        "authorization": "Basic " + os.environ.get("README_API_KEY"),
+    }
+
     # path to the rst file from where we fetch all the RPC commands
     path_to_rst = "doc/index.rst"
     with open(path_to_rst, "r") as file:
         rst_content = file.read()
 
-    commands = extract_rpc_commands(rst_content)
-    if commands:
+    commands_from_local = extract_rpc_commands(rst_content)
+    commands_from_readme = getListOfRPCDocs(headers)
+
+    # Compare local and server commands list to get the list of command to add or delete
+    commands_local_title = set(command[0] for command in commands_from_local)
+    commands_readme_title = set(command['title'] for command in commands_from_readme)
+    commands_to_delete = commands_readme_title - commands_local_title
+    commands_to_add = commands_local_title - commands_readme_title
+    for name in commands_to_delete:
+        publishDoc(Action.DELETE, name, "", 0, headers)
+        sleep(3)
+
+    if commands_from_local:
         order = 0
-        for name, file in commands:
-            print(f"{name}\t\t{file}")
+        for name, file in commands_from_local:
+            # print(f"{name}\t\t{file}")
             with open("doc/" + file) as f:
                 body = f.read()
-                publishDoc(name, body, order)
+                publishDoc(Action.ADD if name in commands_to_add else Action.UPDATE, name, body, order, headers)
             order = order + 1
             sleep(3)
     else:

diff --git a/.github/workflows/bsd.yml b/.github/workflows/bsd.yml
@@ -8,14 +8,14 @@ on:
 
 jobs:
   testfreebsd:
-    runs-on: macos-10.15
+    runs-on: ubuntu-latest
     name: Build and test on FreeBSD
     timeout-minutes: 120
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Test in FreeBSD
       id: test
-      uses: vmactions/freebsd-vm@v0.1.5
+      uses: vmactions/freebsd-vm@v1
       with:
         usesh: true
         prepare: |
@@ -39,12 +39,12 @@ jobs:
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2021-08-3z1
 
           cd /tmp/ || exit 1
-          wget https://bitcoincore.org/bin/bitcoin-core-26.1/bitcoin-26.1-x86_64-linux-gnu.tar.gz
-          tar -xf bitcoin-26.1-x86_64-linux-gnu.tar.bz2
-          sudo mv bitcoin-26.1/bin/* /usr/local/bin
+          wget https://bitcoincore.org/bin/bitcoin-core-27.1/bitcoin-27.1-x86_64-linux-gnu.tar.gz
+          tar -xf bitcoin-27.1-x86_64-linux-gnu.tar.bz2
+          sudo mv bitcoin-27.1/bin/* /usr/local/bin
           rm -rf \
-            bitcoin-26.1-x86_64-linux-gnu.tar.gz \
-            bitcoin-26.1
+            bitcoin-27.1-x86_64-linux-gnu.tar.gz \
+            bitcoin-27.1
           
         run: |
           PATH=/root/.local/bin:$PATH

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -28,7 +28,7 @@ jobs:
       fail-fast: true
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
@@ -43,7 +43,7 @@ jobs:
           git rebase origin/${{ github.base_ref }}
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
@@ -93,10 +93,10 @@ jobs:
             VALGRIND: 0
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
@@ -125,7 +125,7 @@ jobs:
           mv testpack.tar.bz2 cln-${CFG}.tar.bz2
       - name: Check rust packages
         run: cargo test --all
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: cln-${{ matrix.CFG }}.tar.bz2
           path: cln-${{ matrix.CFG }}.tar.bz2
@@ -150,10 +150,10 @@ jobs:
             VALGRIND: 0
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
@@ -170,7 +170,7 @@ jobs:
           git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
 
       - name: Download build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: cln-${{ matrix.CFG }}.tar.bz2
 
@@ -186,10 +186,10 @@ jobs:
       - prebuild
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
@@ -211,8 +211,6 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 120
     env:
-      BITCOIN_VERSION: "26.1"
-      ELEMENTS_VERSION: 23.2.1
       RUST_PROFILE: release  # Has to match the one in the compile step
       PYTEST_OPTS: --timeout=1200 --force-flaky
     needs:
@@ -261,10 +259,10 @@ jobs:
             EXPERIMENTAL_SPLICING: 1
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
@@ -279,7 +277,7 @@ jobs:
         run: .github/scripts/install-bitcoind.sh
 
       - name: Download build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: cln-${{ matrix.CFG }}.tar.bz2
 
@@ -319,8 +317,6 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 120
     env:
-      BITCOIN_VERSION: "26.1"
-      ELEMENTS_VERSION: 23.2.1
       RUST_PROFILE: release  # Has to match the one in the compile step
       CFG: compile-gcc
       PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --force-flaky
@@ -352,10 +348,10 @@ jobs:
             PYTEST_OPTS: --test-group=10 --test-group-count=10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
@@ -370,7 +366,7 @@ jobs:
         run: .github/scripts/install-bitcoind.sh
 
       - name: Download build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: cln-compile-gcc.tar.bz2
 
@@ -389,8 +385,6 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 120
     env:
-      BITCOIN_VERSION: "26.1"
-      ELEMENTS_VERSION: 23.2.1
       RUST_PROFILE: release
       SLOW_MACHINE: 1
       TEST_DEBUG: 1
@@ -423,10 +417,10 @@ jobs:
             PYTEST_OPTS: --test-group=10 --test-group-count=10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
@@ -441,7 +435,7 @@ jobs:
         run: .github/scripts/install-bitcoind.sh
 
       - name: Download build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: cln-compile-clang-sanitizers.tar.bz2