From 241a3c27aea7de33057a4b2314218cea29c82fdd Mon Sep 17 00:00:00 2001 From: agnostic-apollo Date: Mon, 19 Aug 2024 22:23:03 +0500 Subject: [PATCH 1/2] scripts(build-package.sh): move the signing keys info for each repo channel to `repo.json` instead of hardcoding it in build scripts This will allow each repo channel to define its own keys that may be used for verifying signatures of its files. The `termux-packages` fork will just need to add signing keys info in the `repo.json` file itself for any additional channels instead of having to patch build scripts. With the old hardcoded way, the unnecessary pacman key was also being added to local keystore in the current/upstream termux-packages repo, even though no repo channel uses it, so defining the keys in the `repo.json` will solve that issue as well. The `termux_repository__add_repo_signing_keys_to_keystore()` function now handles the logic for adding the repository signing keys to the local keystore. Check its function docs for more info on the new json format and requirements for `repo.json` file. --- build-package.sh | 19 +-- repo.json | 18 ++- scripts/utils/termux/repository/repository.sh | 126 ++++++++++++++++++ 3 files changed, 146 insertions(+), 17 deletions(-) create mode 100644 scripts/utils/termux/repository/repository.sh diff --git a/build-package.sh b/build-package.sh index 53628e7d0050ac..21b339d1d31f54 100755 --- a/build-package.sh +++ b/build-package.sh @@ -30,6 +30,9 @@ source "$TERMUX_SCRIPTDIR/scripts/utils/docker/docker.sh"; docker__create_docker # Functions for working with packages source "$TERMUX_SCRIPTDIR/scripts/utils/package/package.sh" +# Source the repository library. +source "$TERMUX_SCRIPTDIR/scripts/utils/termux/repository/repository.sh" + export SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log -1 --pretty=%ct 2>/dev/null || date "+%s")} if [ "$(uname -o)" = "Android" ] || [ -e "/system/bin/app_process" ]; then @@ -540,20 +543,8 @@ if [ -n "${TERMUX_PACKAGE_LIBRARY-}" ]; then fi if [ "${TERMUX_INSTALL_DEPS-false}" = "true" ] || [ "${TERMUX_PACKAGE_LIBRARY-bionic}" = "glibc" ]; then - # Setup PGP keys for verifying integrity of dependencies. - # Keys are obtained from our keyring package. - gpg --list-keys 2C7F29AE97891F6419A9E2CDB0076E490B71616B > /dev/null 2>&1 || { - gpg --import "$TERMUX_SCRIPTDIR/packages/termux-keyring/grimler.gpg" - gpg --no-tty --command-file <(echo -e "trust\n5\ny") --edit-key 2C7F29AE97891F6419A9E2CDB0076E490B71616B - } - gpg --list-keys CC72CF8BA7DBFA0182877D045A897D96E57CF20C > /dev/null 2>&1 || { - gpg --import "$TERMUX_SCRIPTDIR/packages/termux-keyring/termux-autobuilds.gpg" - gpg --no-tty --command-file <(echo -e "trust\n5\ny") --edit-key CC72CF8BA7DBFA0182877D045A897D96E57CF20C - } - gpg --list-keys 998DE27318E867EA976BA877389CEED64573DFCA > /dev/null 2>&1 || { - gpg --import "$TERMUX_SCRIPTDIR/packages/termux-keyring/termux-pacman.gpg" - gpg --no-tty --command-file <(echo -e "trust\n5\ny") --edit-key 998DE27318E867EA976BA877389CEED64573DFCA - } + # Setup PGP keys for each repo channel in repo.json file for verifying integrity of dependencies. + termux_repository__add_repo_signing_keys_to_keystore "$TERMUX_SCRIPTDIR/repo.json" "$TERMUX_SCRIPTDIR" fi for ((i=0; i<${#PACKAGE_LIST[@]}; i++)); do diff --git a/repo.json b/repo.json index fd6bd5ded48db1..9f5b28d9ad4b3c 100644 --- a/repo.json +++ b/repo.json @@ -4,18 +4,30 @@ "name": "termux-main", "distribution": "stable", "component": "main", - "url": "https://packages-cf.termux.dev/apt/termux-main" + "url": "https://packages-cf.termux.dev/apt/termux-main", + "signing_keys": [ + { "key_file": "packages/termux-keyring/termux-autobuilds.gpg" }, + { "key_file": "packages/termux-keyring/grimler.gpg" } + ] }, "root-packages": { "name": "termux-root", "distribution": "root", "component": "stable", - "url": "https://packages-cf.termux.dev/apt/termux-root" + "url": "https://packages-cf.termux.dev/apt/termux-root", + "signing_keys": [ + { "key_file": "packages/termux-keyring/termux-autobuilds.gpg" }, + { "key_file": "packages/termux-keyring/grimler.gpg" } + ] }, "x11-packages": { "name": "termux-x11", "distribution": "x11", "component": "main", - "url": "https://packages-cf.termux.dev/apt/termux-x11" + "url": "https://packages-cf.termux.dev/apt/termux-x11", + "signing_keys": [ + { "key_file": "packages/termux-keyring/termux-autobuilds.gpg" }, + { "key_file": "packages/termux-keyring/grimler.gpg" } + ] } } diff --git a/scripts/utils/termux/repository/repository.sh b/scripts/utils/termux/repository/repository.sh new file mode 100644 index 00000000000000..667445c6314c41 --- /dev/null +++ b/scripts/utils/termux/repository/repository.sh @@ -0,0 +1,126 @@ +# shellcheck shell=bash + +# Title: repository +# Description: A library for Termux repository utils. + + + +## +# Add repository signing keys to local keystore. +# . +# Each repo channel dict in `repo.json` file should have a +# `signing_keys` key for an array of dicts, where each dict contains +# information for each key that may be required by the parent repo +# channel. At least one signing key info must exist for each repo +# channel. +# . +# The `key_file` key should be set to the path of the key file. If it +# is a relative path that does not start a slash `/`, then +# `termux-packages` repo root directory will be prepended to it. +# . +# +# A dict is being used because other keys may be required +# in future in addition to current `key_file` key, like `key_format`, etc. +# . +# ``` +# "": { +# "signing_keys": [ +# { +# "key_file": "path/to/key1.gpg" +# }, +# { +# "key_file": "path/to/key1.gpg" +# } +# ] +# } +# ``` +# . +# . +# **Parameters:** +# `repo_json_file` - The path to the `repo.json` file. +# `repo_root_dir` - The path to the `termux-packages` repo root +# directory. +# . +# **Returns:** +# Returns `0` if successful, otherwise returns with a non-zero exit code. +# . +# . +# termux_repository__add_repo_signing_keys_to_keystore `` +# `` +## +termux_repository__add_repo_signing_keys_to_keystore() { + + local repo_json_file="$1" + local repo_root_dir="$2" + + local gpg_keys_list + local i + local key_file + local repo_channel_path + local repo_channel_paths_list + local signing_keys_json + local signing_keys_json_array_string + + local -a signing_keys_json_array + + if [[ -z "$repo_json_file" ]]; then + echo "The repo_json_file is not set that is passed to 'termux_repository__add_repo_signing_keys_to_keystore'" 1>&2 + return 1 + elif [[ ! -f "$repo_json_file" ]]; then + echo "The repo_json_file '$repo_json_file' does not exist at path that is passed to 'termux_repository__add_repo_signing_keys_to_keystore'" 1>&2 + return 1 + fi + + # Get the paths list for each repo channel and loop on it. + repo_channel_paths_list="$(jq --raw-output 'del(.pkg_format) | keys | .[]' "$repo_json_file")" || return $? + for repo_channel_path in $repo_channel_paths_list; do + # Check if `signing_keys` array exists for repo channel + if [[ "$(jq ".[\"$repo_channel_path\"].signing_keys"' | if type=="array" then "found" else "not_found" end' "$repo_json_file")" != '"found"' ]]; then + echo "The 'signing_keys' array in '$repo_channel_path' repo channel dict in repo json file '$repo_json_file' does not exist" 1>&2 + echo "Check 'termux_repository__add_repo_signing_keys_to_keystore()' function for more info." 1>&2 + return 1 + fi + + # Get the json for each `dict` in the the `signing_keys` array + # of the repo channel, separated by newlines. + signing_keys_json_array_string="$(jq --compact-output ".[\"$repo_channel_path\"].signing_keys[]" "$repo_json_file")" || return $? + if [[ -z "$signing_keys_json_array_string" ]]; then + echo "The 'signing_keys' array in '$repo_channel_path' repo channel dict in repo json file '$repo_json_file' is empty." 1>&2 + echo "At least one singing key info must exist for a repo channel." 1>&2 + return 1 + fi + + # Create an array of jsons and loop on it. + readarray -t signing_keys_json_array < <(echo "$signing_keys_json_array_string") + i=1 + for signing_keys_json in "${signing_keys_json_array[@]}"; do + key_file=$(echo "$signing_keys_json" | jq --raw-output '.key_file') + + if [[ -z "$key_file" ]] || [[ "$key_file" == "null" ]]; then + echo "The 'key_file' key for signing key $i of '$repo_channel_path' repo channel in repo json file '$repo_json_file' does not exist or is not set" 1>&2 + echo "signing_keys_json: $signing_keys_json" 1>&2 + return 1 + fi + + if [[ "$key_file" != "/"* ]]; then + key_file="$repo_root_dir/$key_file" + fi + + if [[ ! -f "$key_file" ]]; then + echo "The key file '$key_file' for signing key $i of '$repo_channel_path' repo channel in repo json file '$repo_json_file' does not exist at path" 1>&2 + echo "signing_keys_json: $signing_keys_json" 1>&2 + return 1 + fi + + gpg_keys_list=$(gpg --show-keys "$key_file" | sed -n 2p | sed -E -e 's/^[ ]+//') || return $? + gpg --list-keys "$gpg_keys_list" > /dev/null 2>&1 || { + echo "Adding key file '$key_file' for signing key $i of '$repo_channel_path' repo channel in repo json file '$repo_json_file' to gpg keystore" + gpg --import "$key_file" || return $? + gpg --no-tty --command-file <(echo -e "trust\n5\ny") --edit-key "$gpg_keys_list" || return $? + } + + i=$((i + 1)) + done + done + +} From c8600d2037bd7f9882a28d441926fc9ebffa1dcc Mon Sep 17 00:00:00 2001 From: Maxython Date: Wed, 21 Aug 2024 00:07:28 +0300 Subject: [PATCH 2/2] Improving PR The point of this Improving PR is to provide the previous changes in a simpler form, but at the same time maintaining the functionality and capabilities. --- .github/workflows/package_updates.yml | 6 +- .github/workflows/packages.yml | 18 +- build-package.sh | 2 +- repo.json | 57 +++--- scripts/bin/update-checksum | 2 +- scripts/bin/update-packages | 8 +- scripts/lint-packages.sh | 2 +- scripts/properties.sh | 23 ++- scripts/setup-offline-bundle.sh | 2 +- .../utils/termux_pkg_upgrade_version.sh | 2 +- scripts/utils/termux/repository/repository.sh | 171 +++++++++--------- 11 files changed, 149 insertions(+), 144 deletions(-) diff --git a/.github/workflows/package_updates.yml b/.github/workflows/package_updates.yml index 97543468c85685..2b6877683e26e3 100644 --- a/.github/workflows/package_updates.yml +++ b/.github/workflows/package_updates.yml @@ -50,7 +50,7 @@ jobs: echo "Processing pull request #$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH"): ${BASE_COMMIT}..HEAD" CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${BASE_COMMIT}" "HEAD") fi - for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' repo.json); do + for repo_path in $(jq --raw-output '.channels | keys | .[]' repo.json); do repo=$(jq --raw-output '.["'${repo_path}'"].name' repo.json) # Parse changed files and identify new packages and deleted packages. # Create lists of those packages that will be passed to upload job for @@ -83,7 +83,7 @@ jobs: fi done<<<${CHANGED_FILES} done - for repo in $(jq --raw-output 'del(.pkg_format) | .[].name' repo.json); do + for repo in $(jq --raw-output '.channels | .[].name' repo.json); do # Fix so that lists do not contain duplicates if [ -f ./built_${repo}_packages.txt ]; then sort ./built_${repo}_packages.txt | uniq > ./built_${repo}_packages.txt.tmp @@ -107,7 +107,7 @@ jobs: GIT_PUSH_PACKAGES: "false" run: | declare -a packages - for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' repo.json); do + for repo_path in $(jq --raw-output '.channels | keys | .[]' repo.json); do repo=$(jq --raw-output '.["'${repo_path}'"].name' repo.json) if [ -f ./built_${repo}_packages.txt ]; then packages="$packages $(cat ./built_${repo}_packages.txt | tr '\n' ' ')" diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index f34a04814cca37..ebd448e65aa9e4 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -73,7 +73,7 @@ jobs: exit 0 fi - for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' repo.json); do + for repo_path in $(jq --raw-output '.channels | keys | .[]' repo.json); do repo=$(jq --raw-output '.["'${repo_path}'"].name' repo.json) # Parse changed files and identify new packages and deleted packages. # Create lists of those packages that will be passed to upload job for @@ -108,7 +108,7 @@ jobs: done else for pkg in ${{ github.event.inputs.packages }}; do - repo_paths=$(jq --raw-output 'del(.pkg_format) | keys | .[]' repo.json) + repo_paths=$(jq --raw-output '.channels | keys | .[]' repo.json) found=false for repo_path in $repo_paths; do repo=$(jq --raw-output '.["'${repo_path}'"].name' repo.json) @@ -127,7 +127,7 @@ jobs: done fi - for repo in $(jq --raw-output 'del(.pkg_format) | .[].name' repo.json); do + for repo in $(jq --raw-output '.channels | .[].name' repo.json); do # Fix so that lists do not contain duplicates if [ -f ./built_${repo}_packages.txt ]; then sort ./built_${repo}_packages.txt | uniq > ./built_${repo}_packages.txt.tmp @@ -144,7 +144,7 @@ jobs: done declare -a packages - for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' repo.json); do + for repo_path in $(jq --raw-output '.channels | keys | .[]' repo.json); do repo=$(jq --raw-output '.["'${repo_path}'"].name' repo.json) if [ -f ./built_${repo}_packages.txt ]; then packages="$packages $(cat ./built_${repo}_packages.txt | tr '\n' ' ')" @@ -174,7 +174,7 @@ jobs: - name: Lint packages run: | declare -a package_recipes - for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' repo.json); do + for repo_path in $(jq --raw-output '.channels | keys | .[]' repo.json); do repo=$(jq --raw-output '.["'${repo_path}'"].name' repo.json) if [ -f ./built_${repo}_packages.txt ]; then package_recipes="$package_recipes $(cat ./built_${repo}_packages.txt | repo_path=${repo_path} awk '{print ENVIRON["repo_path"]"/"$1"/build.sh"}')" @@ -188,7 +188,7 @@ jobs: - name: Free additional disk space (if needed) run: | declare -a packages - for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' repo.json); do + for repo_path in $(jq --raw-output '.channels | keys | .[]' repo.json); do repo=$(jq --raw-output '.["'${repo_path}'"].name' repo.json) if [ -f ./built_${repo}_packages.txt ]; then packages="$packages $(cat ./built_${repo}_packages.txt | tr '\n' ' ')" @@ -208,7 +208,7 @@ jobs: - name: Build packages run: | declare -a packages - for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' repo.json); do + for repo_path in $(jq --raw-output '.channels | keys | .[]' repo.json); do repo=$(jq --raw-output '.["'${repo_path}'"].name' repo.json) if [ -f ./built_${repo}_packages.txt ]; then packages="$packages $(cat ./built_${repo}_packages.txt | tr '\n' ' ')" @@ -226,7 +226,7 @@ jobs: run: | test -d termux-packages/output && mv termux-packages/output/* ./output/ - for repo in $(jq --raw-output 'del(.pkg_format) | .[].name' repo.json); do + for repo in $(jq --raw-output '.channels | .[].name' repo.json); do # Put package lists into directory with *.deb files so they will be transferred to # upload job. test -f ./built_${repo}_packages.txt && mv ./built_${repo}_packages.txt ./debs/ @@ -290,7 +290,7 @@ jobs: tar xf "$archive" done - for repo in $(jq --raw-output 'del(.pkg_format) | keys | .[]' repo.json); do + for repo in $(jq --raw-output '.channels | keys | .[]' repo.json); do export REPOSITORY_NAME=$(jq --raw-output '.["'$repo'"].name' repo.json) export REPOSITORY_DISTRIBUTION=$(jq --raw-output '.["'$repo'"].distribution' repo.json) diff --git a/build-package.sh b/build-package.sh index 21b339d1d31f54..1bec3ac369ef0a 100755 --- a/build-package.sh +++ b/build-package.sh @@ -544,7 +544,7 @@ fi if [ "${TERMUX_INSTALL_DEPS-false}" = "true" ] || [ "${TERMUX_PACKAGE_LIBRARY-bionic}" = "glibc" ]; then # Setup PGP keys for each repo channel in repo.json file for verifying integrity of dependencies. - termux_repository__add_repo_signing_keys_to_keystore "$TERMUX_SCRIPTDIR/repo.json" "$TERMUX_SCRIPTDIR" + termux_repository__add_repo_signing_keys_to_keystore fi for ((i=0; i<${#PACKAGE_LIST[@]}; i++)); do diff --git a/repo.json b/repo.json index 9f5b28d9ad4b3c..08848d4aca696c 100644 --- a/repo.json +++ b/repo.json @@ -1,33 +1,34 @@ { "pkg_format": "debian", - "packages": { - "name": "termux-main", - "distribution": "stable", - "component": "main", - "url": "https://packages-cf.termux.dev/apt/termux-main", - "signing_keys": [ - { "key_file": "packages/termux-keyring/termux-autobuilds.gpg" }, - { "key_file": "packages/termux-keyring/grimler.gpg" } - ] + "signing_keys": { + "termux": { + "key_file": "packages/termux-keyring/termux-autobuilds.gpg" + }, + "grimler": { + "key_file": "packages/termux-keyring/grimler.gpg" + } }, - "root-packages": { - "name": "termux-root", - "distribution": "root", - "component": "stable", - "url": "https://packages-cf.termux.dev/apt/termux-root", - "signing_keys": [ - { "key_file": "packages/termux-keyring/termux-autobuilds.gpg" }, - { "key_file": "packages/termux-keyring/grimler.gpg" } - ] - }, - "x11-packages": { - "name": "termux-x11", - "distribution": "x11", - "component": "main", - "url": "https://packages-cf.termux.dev/apt/termux-x11", - "signing_keys": [ - { "key_file": "packages/termux-keyring/termux-autobuilds.gpg" }, - { "key_file": "packages/termux-keyring/grimler.gpg" } - ] + "channels": { + "packages": { + "name": "termux-main", + "distribution": "stable", + "component": "main", + "url": "https://packages-cf.termux.dev/apt/termux-main", + "keys": ["termux", "grimler"] + }, + "root-packages": { + "name": "termux-root", + "distribution": "root", + "component": "stable", + "url": "https://packages-cf.termux.dev/apt/termux-root", + "keys": ["termux", "grimler"] + }, + "x11-packages": { + "name": "termux-x11", + "distribution": "x11", + "component": "main", + "url": "https://packages-cf.termux.dev/apt/termux-x11", + "keys": ["termux", "grimler"] + } } } diff --git a/scripts/bin/update-checksum b/scripts/bin/update-checksum index 19d79754c3b4d7..0eb4b08d65148c 100755 --- a/scripts/bin/update-checksum +++ b/scripts/bin/update-checksum @@ -39,7 +39,7 @@ for package in "${@}"; do buildsh_path="${package}/build.sh" package=$(basename ${package}) else - for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' $REPO_ROOT/repo.json); do + for repo_path in $(jq --raw-output '.channels | keys | .[]' $REPO_ROOT/repo.json); do if [ -d "${repo_path}/${package}" ] && [ -f "${repo_path}/${package}/build.sh" ]; then repo=$(jq --raw-output '.["'$repo_path'"].name' $REPO_ROOT/repo.json) repo=${repo#"termux-"} diff --git a/scripts/bin/update-packages b/scripts/bin/update-packages index 366f7c7c78b54e..90781fd7ac1cd8 100755 --- a/scripts/bin/update-packages +++ b/scripts/bin/update-packages @@ -25,7 +25,7 @@ export TERMUX_SCRIPTDIR TERMUX_SCRIPTDIR=$(realpath "$(dirname "$0")/../..") # Root of repository. export TERMUX_PACKAGES_DIRECTORIES -TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}"/repo.json) +TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output '.channels | keys | .[]' "${TERMUX_SCRIPTDIR}"/repo.json) # Define few more variables used by scripts. # shellcheck source=scripts/properties.sh @@ -151,7 +151,7 @@ _fetch_and_cache_tags() { quiet termux_repology_api_get_latest_version ' ' local __PACKAGES=() - for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do + for repo_dir in $(jq --raw-output '.channels | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do for pkg_dir in "${repo_dir}"/*; do ! quiet _should_update "${pkg_dir}" && continue # Skip if not needed. grep -q '^termux_pkg_auto_update' "${pkg_dir}/build.sh" && continue # Skip if package has custom auto-update @@ -310,7 +310,7 @@ echo "INFO: Running update for: $*" if [[ "$1" == "@all" ]]; then _fetch_and_cache_tags - for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do + for repo_dir in $(jq --raw-output '.channels | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do for pkg_dir in "${repo_dir}"/*; do _unix_millis=$(date +%s%N | cut -b1-13) ! _should_update "${pkg_dir}" && continue # Skip if not needed. @@ -330,7 +330,7 @@ if [[ "$1" == "@all" ]]; then else for pkg in "$@"; do if [ ! -d "${pkg}" ]; then # If only package name is given, try to find it's directory. - for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do + for repo_dir in $(jq --raw-output '.channels | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do if [ -d "${repo_dir}/${pkg}" ]; then pkg="${repo_dir}/${pkg}" break diff --git a/scripts/lint-packages.sh b/scripts/lint-packages.sh index 7b07fdb44f0ccd..3126c30623984c 100755 --- a/scripts/lint-packages.sh +++ b/scripts/lint-packages.sh @@ -592,7 +592,7 @@ linter_main() { if (( $# )); then linter_main "$@" || exit 1 else - for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' $TERMUX_SCRIPTDIR/repo.json); do + for repo_dir in $(jq --raw-output '.channels | keys | .[]' $TERMUX_SCRIPTDIR/repo.json); do linter_main $repo_dir/*/build.sh done || exit 1 fi diff --git a/scripts/properties.sh b/scripts/properties.sh index 03b87e35fd2969..8f510c8617849a 100644 --- a/scripts/properties.sh +++ b/scripts/properties.sh @@ -55,17 +55,22 @@ TERMUX_REPO_PACKAGE="com.termux" TERMUX_REPO_URL=() TERMUX_REPO_DISTRIBUTION=() TERMUX_REPO_COMPONENT=() +TERMUX_REPO_KEYS=() +TERMUX_SIGNING_KEYS_FILE=() +#TERMUX_SIGNING_KEYS_FORMAT=() -export TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' ${TERMUX_SCRIPTDIR}/repo.json) - -for url in $(jq -r 'del(.pkg_format) | .[] | .url' ${TERMUX_SCRIPTDIR}/repo.json); do - TERMUX_REPO_URL+=("$url") -done -for distribution in $(jq -r 'del(.pkg_format) | .[] | .distribution' ${TERMUX_SCRIPTDIR}/repo.json); do - TERMUX_REPO_DISTRIBUTION+=("$distribution") +export TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output '.channels | keys | join(" ")' ${TERMUX_SCRIPTDIR}/repo.json) +for pkg_dir in $TERMUX_PACKAGES_DIRECTORIES; do + TERMUX_REPO_URL+=("$(jq -r '.channels | ."'${pkg_dir}'" | .url' ${TERMUX_SCRIPTDIR}/repo.json)") + TERMUX_REPO_DISTRIBUTION+=("$(jq -r '.channels | ."'${pkg_dir}'" | .distribution' ${TERMUX_SCRIPTDIR}/repo.json)") + TERMUX_REPO_COMPONENT+=("$(jq -r '.channels | ."'${pkg_dir}'" | .component' ${TERMUX_SCRIPTDIR}/repo.json)") + TERMUX_REPO_KEYS+=("$(jq -r '.channels | ."'${pkg_dir}'" | .keys | if . != null then join(" ") else . end' ${TERMUX_SCRIPTDIR}/repo.json)") done -for component in $(jq -r 'del(.pkg_format) | .[] | .component' ${TERMUX_SCRIPTDIR}/repo.json); do - TERMUX_REPO_COMPONENT+=("$component") + +export TERMUX_SIGNING_KEYS=$(jq --raw-output '.signing_keys | keys | join(" ")' ${TERMUX_SCRIPTDIR}/repo.json) +for signing_key in $TERMUX_SIGNING_KEYS; do + TERMUX_SIGNING_KEYS_FILE+=("$(jq -r '.signing_keys | ."'${signing_key}'" | .key_file' ${TERMUX_SCRIPTDIR}/repo.json)") + #TERMUX_SIGNING_KEYS_FORMAT+=("$(jq -r '.signing_keys | ."'${signing_key}'" | .key_format' ${TERMUX_SCRIPTDIR}/repo.json)") done # Allow to override setup. diff --git a/scripts/setup-offline-bundle.sh b/scripts/setup-offline-bundle.sh index 8433d185b3c4b7..505d0393eef3f7 100755 --- a/scripts/setup-offline-bundle.sh +++ b/scripts/setup-offline-bundle.sh @@ -78,7 +78,7 @@ mkdir -p "$TERMUX_PKG_TMPDIR" rm -rf "${TERMUX_PKG_TMPDIR}" # Package sources. -for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' $TERMUX_SCRIPTDIR/repo.json); do +for repo_path in $(jq --raw-output '.channels | keys | .[]' $TERMUX_SCRIPTDIR/repo.json); do for p in "$TERMUX_SCRIPTDIR"/$repo_path/*; do ( . "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_step_get_source.sh diff --git a/scripts/updates/utils/termux_pkg_upgrade_version.sh b/scripts/updates/utils/termux_pkg_upgrade_version.sh index 8278883f315235..d2e7f37973d705 100755 --- a/scripts/updates/utils/termux_pkg_upgrade_version.sh +++ b/scripts/updates/utils/termux_pkg_upgrade_version.sh @@ -84,7 +84,7 @@ termux_pkg_upgrade_version() { echo "INFO: Trying to build package." - for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' ${TERMUX_SCRIPTDIR}/repo.json); do + for repo_path in $(jq --raw-output '.channels | keys | .[]' ${TERMUX_SCRIPTDIR}/repo.json); do _buildsh_path="${TERMUX_SCRIPTDIR}/${repo_path}/${TERMUX_PKG_NAME}/build.sh" repo=$(jq --raw-output ".\"${repo_path}\".name" ${TERMUX_SCRIPTDIR}/repo.json) repo=${repo#"termux-"} diff --git a/scripts/utils/termux/repository/repository.sh b/scripts/utils/termux/repository/repository.sh index 667445c6314c41..7a85d7eab5dfce 100644 --- a/scripts/utils/termux/repository/repository.sh +++ b/scripts/utils/termux/repository/repository.sh @@ -5,122 +5,121 @@ +## +# Check that each repos specifies existing signing key names. +# . +# Each repo channel must contain a `key` key, which must store +# a list of signing key names in the `repo.json` file. The names +# of the signing keys are the keys of the `signing_keys` key, +# which are configured separately. +# . +# ``` +# "": { +# "keys": ["signing_key1", "signing_key2"] +# } +# ``` +# . +# . +# **Parameters:** +# `TERMUX_PACKAGES_DIRECTORIES` - list of package directories (repos). +# `TERMUX_REPO_KEYS` - list of signing key names for each repos. +# . +# **Returns:** +# Returns 0 if everything is ok. Otherwise, an error. +# . +# . +# TERMUX_PACKAGES_DIRECTORIES= TERMUX_REPO_KEYS= termux_repository__check_repo_keys_name +## +termux_repository__check_repo_keys_name() { + local index=0 + local repo + local key + for repo in ${TERMUX_PACKAGES_DIRECTORIES}; do + for key in ${TERMUX_REPO_KEYS[$index]}; do + if [[ -z "$key" ]] || [[ "$key" == "null" ]]; then + echo "The repo channel '${repo}' does not specify a list of signing keys in repo json file" 1>&2 + return 1 + fi + if ! [[ " ${TERMUX_SIGNING_KEYS} " =~ " ${key} " ]]; then + echo "The repo channel '${repo}' specifies non-existent signing key '${key}' in repo json file" 1>&2 + return 1 + fi + done + index=$((index + 1)) + done + return 0 +} + + + ## # Add repository signing keys to local keystore. # . -# Each repo channel dict in `repo.json` file should have a -# `signing_keys` key for an array of dicts, where each dict contains -# information for each key that may be required by the parent repo -# channel. At least one signing key info must exist for each repo -# channel. +# To store signing keys, the `signing_keys` key of the `repo.json` +# file is used. To configure (add) a signing key, you first need to +# give a name to the signing key, which will be the key of `signing_keys`, +# and then configure the following keys inside your signing key: `key_file`. # . -# The `key_file` key should be set to the path of the key file. If it -# is a relative path that does not start a slash `/`, then -# `termux-packages` repo root directory will be prepended to it. +# `key_file` - path to signing key file (full path or not full +# but accessible via path `TERMUX_SCRIPTDIR`). # . -# # A dict is being used because other keys may be required # in future in addition to current `key_file` key, like `key_format`, etc. # . # ``` -# "": { -# "signing_keys": [ -# { -# "key_file": "path/to/key1.gpg" -# }, -# { +# "signing_keys": { +# "signing_key1": { # "key_file": "path/to/key1.gpg" -# } -# ] +# }, +# "signing_key2": { +# "key_file": "path/to/key2.gpg" +# } # } # ``` # . # . # **Parameters:** -# `repo_json_file` - The path to the `repo.json` file. -# `repo_root_dir` - The path to the `termux-packages` repo root -# directory. +# `TERMUX_SIGNING_KEYS` - list of signing key names. +# `TERMUX_SIGNING_KEYS_FILE` - list of paths to the signing key file. +# `TERMUX_SCRIPTDIR` - full path to the repository directory. # . # **Returns:** # Returns `0` if successful, otherwise returns with a non-zero exit code. # . # . -# termux_repository__add_repo_signing_keys_to_keystore `` -# `` +# TERMUX_SIGNING_KEYS= TERMUX_SIGNING_KEYS_FILE= TERMUX_SCRIPTDIR= termux_repository__add_repo_signing_keys_to_keystore ## termux_repository__add_repo_signing_keys_to_keystore() { + # The first step is to check the specified key names in repos. + termux_repository__check_repo_keys_name - local repo_json_file="$1" - local repo_root_dir="$2" - + # The second stage is adding signing keys. + local sig_keys_array=(${TERMUX_SIGNING_KEYS}) local gpg_keys_list - local i - local key_file - local repo_channel_path - local repo_channel_paths_list - local signing_keys_json - local signing_keys_json_array_string + local index + for index in ${!sig_keys_array[@]}; do + local sig_key="${sig_keys_array[$index]}" + local key_file="${TERMUX_SIGNING_KEYS_FILE[$index]}" - local -a signing_keys_json_array - - if [[ -z "$repo_json_file" ]]; then - echo "The repo_json_file is not set that is passed to 'termux_repository__add_repo_signing_keys_to_keystore'" 1>&2 - return 1 - elif [[ ! -f "$repo_json_file" ]]; then - echo "The repo_json_file '$repo_json_file' does not exist at path that is passed to 'termux_repository__add_repo_signing_keys_to_keystore'" 1>&2 - return 1 - fi - - # Get the paths list for each repo channel and loop on it. - repo_channel_paths_list="$(jq --raw-output 'del(.pkg_format) | keys | .[]' "$repo_json_file")" || return $? - for repo_channel_path in $repo_channel_paths_list; do - # Check if `signing_keys` array exists for repo channel - if [[ "$(jq ".[\"$repo_channel_path\"].signing_keys"' | if type=="array" then "found" else "not_found" end' "$repo_json_file")" != '"found"' ]]; then - echo "The 'signing_keys' array in '$repo_channel_path' repo channel dict in repo json file '$repo_json_file' does not exist" 1>&2 - echo "Check 'termux_repository__add_repo_signing_keys_to_keystore()' function for more info." 1>&2 + if [[ -z "$key_file" ]] || [[ "$key_file" == "null" ]]; then + echo "The 'key_file' key for signing key $sig_key in repo json file does not exist or is not set" 1>&2 return 1 fi - # Get the json for each `dict` in the the `signing_keys` array - # of the repo channel, separated by newlines. - signing_keys_json_array_string="$(jq --compact-output ".[\"$repo_channel_path\"].signing_keys[]" "$repo_json_file")" || return $? - if [[ -z "$signing_keys_json_array_string" ]]; then - echo "The 'signing_keys' array in '$repo_channel_path' repo channel dict in repo json file '$repo_json_file' is empty." 1>&2 - echo "At least one singing key info must exist for a repo channel." 1>&2 - return 1 + if [[ "$key_file" != "/"* ]]; then + key_file="$TERMUX_SCRIPTDIR/$key_file" fi - # Create an array of jsons and loop on it. - readarray -t signing_keys_json_array < <(echo "$signing_keys_json_array_string") - i=1 - for signing_keys_json in "${signing_keys_json_array[@]}"; do - key_file=$(echo "$signing_keys_json" | jq --raw-output '.key_file') - - if [[ -z "$key_file" ]] || [[ "$key_file" == "null" ]]; then - echo "The 'key_file' key for signing key $i of '$repo_channel_path' repo channel in repo json file '$repo_json_file' does not exist or is not set" 1>&2 - echo "signing_keys_json: $signing_keys_json" 1>&2 - return 1 - fi - - if [[ "$key_file" != "/"* ]]; then - key_file="$repo_root_dir/$key_file" - fi - - if [[ ! -f "$key_file" ]]; then - echo "The key file '$key_file' for signing key $i of '$repo_channel_path' repo channel in repo json file '$repo_json_file' does not exist at path" 1>&2 - echo "signing_keys_json: $signing_keys_json" 1>&2 - return 1 - fi - - gpg_keys_list=$(gpg --show-keys "$key_file" | sed -n 2p | sed -E -e 's/^[ ]+//') || return $? - gpg --list-keys "$gpg_keys_list" > /dev/null 2>&1 || { - echo "Adding key file '$key_file' for signing key $i of '$repo_channel_path' repo channel in repo json file '$repo_json_file' to gpg keystore" - gpg --import "$key_file" || return $? - gpg --no-tty --command-file <(echo -e "trust\n5\ny") --edit-key "$gpg_keys_list" || return $? - } + if [[ ! -f "$key_file" ]]; then + echo "The key file '$key_file' for signing key $sig_key in repo json file does not exist at path" 1>&2 + return 1 + fi - i=$((i + 1)) - done + gpg_keys_list=$(gpg --show-keys "$key_file" | sed -n 2p | sed -E -e 's/^[ ]+//') || return $? + gpg --list-keys "$gpg_keys_list" > /dev/null 2>&1 || { + echo "Adding key file '$key_file' for signing key $sig_key in repo json file to gpg keystore" + gpg --import "$key_file" || return $? + gpg --no-tty --command-file <(echo -e "trust\n5\ny") --edit-key "$gpg_keys_list" || return $? + } done - }