From 34e554f328e26f5c015f59304ab58f7a310e9490 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Sun, 28 Jul 2024 17:54:55 -0700 Subject: [PATCH 01/12] [terminfo-db] Add a builder for the terminfo database This uses the directory-based database rather than the single Berkeley database (terminfo.db). The license is from ncurses, which is close enough; see https://invisible-island.net/ncurses/terminfo.src.html. Ideally this would be shipped as one of Julia's dependencies to ensure that the custom terminfo parser actually works on all platforms, as not all have the terminfo database. --- T/TermInfoDB/build_tarballs.jl | 70 ++++++++++++++++++++++++++++++++++ T/TermInfoDB/bundled/COPYING | 29 ++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 T/TermInfoDB/build_tarballs.jl create mode 100644 T/TermInfoDB/bundled/COPYING diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl new file mode 100644 index 00000000000..0d2da2fc1b3 --- /dev/null +++ b/T/TermInfoDB/build_tarballs.jl @@ -0,0 +1,70 @@ +using BinaryBuilder + +# This is based on https://www.freshports.org/misc/terminfo-db/ +name = "TermInfoDB" +version = v"2023.12.9" + +v = string(version.major, lpad(version.minor, 2, '0'), lpad(version.patch, 2, '0')) +sources = [ + FileSource("https://invisible-island.net/archives/ncurses/current/terminfo-$v.src.gz", + "2debcf2fd689988d44558bcd8a26a104b96542ffc9540f19e2586b3aeecd1c79"), + DirectorySource("./bundled"), +] + +script = raw""" +cd ${WORKSPACE}/srcdir/ +gunzip terminfo-* +tic -sx -o "${prefix}/share/terminfo" ./terminfo-* + +# When ignoring case, these files are duplicates of others. We'll remove them to ensure +# they don't cause trouble on case-insensitive filesystems +DUPS=( + 2/2621a + e/eterm + e/eterm-color + h/hp2621a + h/hp70092a + l/lft-pc850 + n/ncr260vt300wpp + n/ncrvt100wpp + p/p12 + p/p12-m + p/p12-m-w + p/p12-w + p/p14 + p/p14-m + p/p14-m-w + p/p14-w + p/p4 + p/p5 + p/p7 + p/p8 + p/p8-w + p/p9 + p/p9-8 + p/p9-8-w + p/p9-w +) +for file in "${DUPS[@]}"; do + rm -f "${prefix}/share/terminfo/${file}" +done +# Remove empty directories +find "${prefix}/share/terminfo/" -type d -empty -delete + +install_license "${WORKSPACE}/srcdir/COPYING" +""" + +platforms = supported_platforms() + +# I'd rather not list out >2k individual `FileProduct`s so let's just call the entry +# for xterm our "product" and hope the rest are there +products = [ + FileProduct("share/terminfo/x/xterm", :terminfo_xterm), +] + +dependencies = [ + HostBuildDependency("Ncurses_jll"), # for `tic` +] + +build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; + julia_compat="1.6") diff --git a/T/TermInfoDB/bundled/COPYING b/T/TermInfoDB/bundled/COPYING new file mode 100644 index 00000000000..2abcc60d6cd --- /dev/null +++ b/T/TermInfoDB/bundled/COPYING @@ -0,0 +1,29 @@ +Copyright 2018-2022,2023 Thomas E. Dickey +Copyright 1998-2017,2018 Free Software Foundation, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, distribute with modifications, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE ABOVE COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name(s) of the above copyright +holders shall not be used in advertising or otherwise to promote the +sale, use or other dealings in this Software without prior written +authorization. + +-- vile:txtmode fc=72 +-- $Id: COPYING,v 1.12 2023/01/07 17:55:53 tom Exp $ From aed9138c00eea4fc42ad7eddbd8d15c3f1ad8717 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 29 Jul 2024 08:54:55 -0700 Subject: [PATCH 02/12] [terminfo-db] mkdir -p --- T/TermInfoDB/build_tarballs.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index 0d2da2fc1b3..12d2b20a274 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -14,6 +14,7 @@ sources = [ script = raw""" cd ${WORKSPACE}/srcdir/ gunzip terminfo-* +mkdir -p "${prefix}/share/terminfo" tic -sx -o "${prefix}/share/terminfo" ./terminfo-* # When ignoring case, these files are duplicates of others. We'll remove them to ensure From 04c7b3bd7eb20e4519fdefa35c947e6e1dd93e16 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 29 Jul 2024 09:46:31 -0700 Subject: [PATCH 03/12] [terminfo-db] Use `AnyPlatform` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mosè Giordano --- T/TermInfoDB/build_tarballs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index 12d2b20a274..03bf5cada84 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -55,7 +55,7 @@ find "${prefix}/share/terminfo/" -type d -empty -delete install_license "${WORKSPACE}/srcdir/COPYING" """ -platforms = supported_platforms() +platforms = [AnyPlatform()] # I'd rather not list out >2k individual `FileProduct`s so let's just call the entry # for xterm our "product" and hope the rest are there From 4fa7698b43773fd5142e6c520139b85a4a60ecd7 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 29 Jul 2024 16:09:18 -0700 Subject: [PATCH 04/12] Make logging more verbose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mosè Giordano --- T/TermInfoDB/build_tarballs.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index 03bf5cada84..9935e2e571d 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -47,10 +47,10 @@ DUPS=( p/p9-w ) for file in "${DUPS[@]}"; do - rm -f "${prefix}/share/terminfo/${file}" + rm -fv "${prefix}/share/terminfo/${file}" done # Remove empty directories -find "${prefix}/share/terminfo/" -type d -empty -delete +find "${prefix}/share/terminfo/" -type d -empty -print -delete install_license "${WORKSPACE}/srcdir/COPYING" """ From e030cbfe3f94abff67bd56c576e7f069945b810f Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 29 Jul 2024 18:02:13 -0700 Subject: [PATCH 05/12] [terminfo-db] Try a more automated approach to deduplication --- T/TermInfoDB/build_tarballs.jl | 53 +++++++++++++--------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index 9935e2e571d..e88455321c7 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -17,40 +17,27 @@ gunzip terminfo-* mkdir -p "${prefix}/share/terminfo" tic -sx -o "${prefix}/share/terminfo" ./terminfo-* -# When ignoring case, these files are duplicates of others. We'll remove them to ensure -# they don't cause trouble on case-insensitive filesystems -DUPS=( - 2/2621a - e/eterm - e/eterm-color - h/hp2621a - h/hp70092a - l/lft-pc850 - n/ncr260vt300wpp - n/ncrvt100wpp - p/p12 - p/p12-m - p/p12-m-w - p/p12-w - p/p14 - p/p14-m - p/p14-m-w - p/p14-w - p/p4 - p/p5 - p/p7 - p/p8 - p/p8-w - p/p9 - p/p9-8 - p/p9-8-w - p/p9-w -) -for file in "${DUPS[@]}"; do - rm -fv "${prefix}/share/terminfo/${file}" +# The terminfo filesystem-based database contains both upper- and lowercase directory +# names, which presents a problem for case-insensitive filesystems. Let's rename all +# files and directories to lowercase, dealing with conflicts by overwriting and hoping +# for the best. +pushd "${prefix}/share/terminfo/" +for dir in $(ls); do + if [[ ${dir} =~ [A-Z]+ ]] && [[ -d ${dir,,} ]]; then + for file in $(ls ${dir}); do + mv -fv "${dir}/${file}" "${dir,,}/${file,,}" + done + if [ -z "$(ls -A "${dir}")" ]; then + rm -rfv "${dir}" + fi + elif [[ ${dir} =~ \d+ ]]; then + for file in $(ls ${dir}); do + mv -fv "${dir}/${file}" "${dir}/${file,,}" + done + fi done -# Remove empty directories -find "${prefix}/share/terminfo/" -type d -empty -print -delete +tree +popd install_license "${WORKSPACE}/srcdir/COPYING" """ From 6c82cbcfdf7f2ee28eafb067a0f5792f4900f0fd Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 29 Jul 2024 18:10:50 -0700 Subject: [PATCH 06/12] [terminfo-db] Install `tree` for the lulz --- T/TermInfoDB/build_tarballs.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index e88455321c7..eea85d2fde3 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -13,6 +13,11 @@ sources = [ script = raw""" cd ${WORKSPACE}/srcdir/ + +# Install `tree` to get an overview of what's going to be included in the JLL +apk update +apk add tree + gunzip terminfo-* mkdir -p "${prefix}/share/terminfo" tic -sx -o "${prefix}/share/terminfo" ./terminfo-* From 2ac18b0096a6928e4f6c4b5f990d6f3f10a96264 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 29 Jul 2024 18:22:24 -0700 Subject: [PATCH 07/12] [terminfo-db] Do a better job with the renaming thing --- T/TermInfoDB/build_tarballs.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index eea85d2fde3..a91727af430 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -28,17 +28,17 @@ tic -sx -o "${prefix}/share/terminfo" ./terminfo-* # for the best. pushd "${prefix}/share/terminfo/" for dir in $(ls); do - if [[ ${dir} =~ [A-Z]+ ]] && [[ -d ${dir,,} ]]; then - for file in $(ls ${dir}); do - mv -fv "${dir}/${file}" "${dir,,}/${file,,}" - done - if [ -z "$(ls -A "${dir}")" ]; then - rm -rfv "${dir}" + lcdir="${dir,,}" + for file in $(ls ${dir}); do + lcfile="${file,,}" + if [ "${dir}/${file}" = "${lcdir}/${lcfile}" ]; then + # Already all lowercase, nothing to do + continue fi - elif [[ ${dir} =~ \d+ ]]; then - for file in $(ls ${dir}); do - mv -fv "${dir}/${file}" "${dir}/${file,,}" - done + mv -fv "${dir}/${file}" "${lcdir}/${lcfile}" + done + if [ -z "$(ls -A "${dir}")" ]; then + rm -rfv "${dir}" fi done tree From 6b3ba5773b81a205e2e96e65e7d209a1d2016793 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Wed, 31 Jul 2024 08:52:36 -0700 Subject: [PATCH 08/12] [terminfo-db] debug --- T/TermInfoDB/build_tarballs.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index a91727af430..e885470a0d7 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -36,6 +36,10 @@ for dir in $(ls); do continue fi mv -fv "${dir}/${file}" "${lcdir}/${lcfile}" + if [ -e "${dir}/${file}" ]; then + echo "ERROR: '${dir}/${file}' not successfully renamed to lowercase!!!" + exit 1 + fi done if [ -z "$(ls -A "${dir}")" ]; then rm -rfv "${dir}" From 362abf842a4ebb2dce6a91fc67b2b7bfe8dee23e Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 5 Aug 2024 14:47:47 -0700 Subject: [PATCH 09/12] Computer is mistaek --- T/TermInfoDB/build_tarballs.jl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index e885470a0d7..dd8cead31a2 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -14,6 +14,9 @@ sources = [ script = raw""" cd ${WORKSPACE}/srcdir/ +# Adopting the wisdom of https://mywiki.wooledge.org/ParsingLs (🙏 Mosè) +shopt -s nullglob + # Install `tree` to get an overview of what's going to be included in the JLL apk update apk add tree @@ -27,16 +30,18 @@ tic -sx -o "${prefix}/share/terminfo" ./terminfo-* # files and directories to lowercase, dealing with conflicts by overwriting and hoping # for the best. pushd "${prefix}/share/terminfo/" -for dir in $(ls); do +for dir in *; do lcdir="${dir,,}" - for file in $(ls ${dir}); do + for file in ${dir}/*; do lcfile="${file,,}" if [ "${dir}/${file}" = "${lcdir}/${lcfile}" ]; then # Already all lowercase, nothing to do continue fi - mv -fv "${dir}/${file}" "${lcdir}/${lcfile}" - if [ -e "${dir}/${file}" ]; then + mv -fv "${dir}/${file}" "${lcdir}/${lcfile}.TEMP" + rm -fv "${lcdir}/${lcfile}" + mv -fv "${lcdir}/${lcfile}.TEMP" "${lcdir}/${lcfile}" + if [ -e "${dir}/${file}" ] || [ -e "${lcdir}/${lcfile}.TEMP" ]; then echo "ERROR: '${dir}/${file}' not successfully renamed to lowercase!!!" exit 1 fi From 08ee6ed3227697b45d1a5870a2429737093d1d94 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 5 Aug 2024 14:56:41 -0700 Subject: [PATCH 10/12] Do I need to make sure the `lcdir` exists? Perhaps I do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mosè Giordano --- T/TermInfoDB/build_tarballs.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index dd8cead31a2..375ea06a85e 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -32,6 +32,7 @@ tic -sx -o "${prefix}/share/terminfo" ./terminfo-* pushd "${prefix}/share/terminfo/" for dir in *; do lcdir="${dir,,}" + mkdir -vp "${lcdir}" for file in ${dir}/*; do lcfile="${file,,}" if [ "${dir}/${file}" = "${lcdir}/${lcfile}" ]; then From 68d1fbb864dd97e056578ef70ee4d5ff9d977103 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 5 Aug 2024 15:15:48 -0700 Subject: [PATCH 11/12] based MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mosè Giordano --- T/TermInfoDB/build_tarballs.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index 375ea06a85e..4d1865d2a9a 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -34,6 +34,7 @@ for dir in *; do lcdir="${dir,,}" mkdir -vp "${lcdir}" for file in ${dir}/*; do + file=$(basename "${file}") lcfile="${file,,}" if [ "${dir}/${file}" = "${lcdir}/${lcfile}" ]; then # Already all lowercase, nothing to do From 78ccc23654dc78f5cf1d42919767bc740af7a51b Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 5 Aug 2024 18:57:42 -0700 Subject: [PATCH 12/12] Be less verbose in output, more verbose in comments --- T/TermInfoDB/build_tarballs.jl | 36 ++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/T/TermInfoDB/build_tarballs.jl b/T/TermInfoDB/build_tarballs.jl index 4d1865d2a9a..75b7ac95ccb 100644 --- a/T/TermInfoDB/build_tarballs.jl +++ b/T/TermInfoDB/build_tarballs.jl @@ -17,22 +17,22 @@ cd ${WORKSPACE}/srcdir/ # Adopting the wisdom of https://mywiki.wooledge.org/ParsingLs (🙏 Mosè) shopt -s nullglob -# Install `tree` to get an overview of what's going to be included in the JLL -apk update -apk add tree - gunzip terminfo-* mkdir -p "${prefix}/share/terminfo" tic -sx -o "${prefix}/share/terminfo" ./terminfo-* +pushd "${prefix}/share/terminfo/" + # The terminfo filesystem-based database contains both upper- and lowercase directory # names, which presents a problem for case-insensitive filesystems. Let's rename all -# files and directories to lowercase, dealing with conflicts by overwriting and hoping -# for the best. -pushd "${prefix}/share/terminfo/" +# files and directories to lowercase. Unfortunately, we can't just `mv` the filename +# to its lowercase counterpart because many of these files, including but not limited +# to those whose names differ only by case, are actually hard links to one another +# (they have the same inode) so `mv` sees it as a no-op and just does nothing. +# Hence... this. for dir in *; do lcdir="${dir,,}" - mkdir -vp "${lcdir}" + mkdir -p "${lcdir}" for file in ${dir}/*; do file=$(basename "${file}") lcfile="${file,,}" @@ -40,19 +40,29 @@ for dir in *; do # Already all lowercase, nothing to do continue fi - mv -fv "${dir}/${file}" "${lcdir}/${lcfile}.TEMP" - rm -fv "${lcdir}/${lcfile}" - mv -fv "${lcdir}/${lcfile}.TEMP" "${lcdir}/${lcfile}" + mv -f "${dir}/${file}" "${lcdir}/${lcfile}.TEMP" + rm -f "${lcdir}/${lcfile}" + mv -f "${lcdir}/${lcfile}.TEMP" "${lcdir}/${lcfile}" if [ -e "${dir}/${file}" ] || [ -e "${lcdir}/${lcfile}.TEMP" ]; then echo "ERROR: '${dir}/${file}' not successfully renamed to lowercase!!!" exit 1 fi done if [ -z "$(ls -A "${dir}")" ]; then - rm -rfv "${dir}" + rm -rf "${dir}" fi done -tree + +# I'm not about to list the entire contents out as `FileProduct`s, so we'll do our +# own mini-audit by checking that the expected directories exist. We know each is +# non-empty based on the above. +dirs=(*) +expected=(1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x z) +if [ ! -z "$(echo ${dirs[@]} ${expected[@]} | tr ' ' '\n' | sort | uniq -u)" ]; then + echo "ERROR: Build did not produce the expected set of directories!!!" + exit 1 +fi + popd install_license "${WORKSPACE}/srcdir/COPYING"