From 3530342dccf37c70dbf8b3b20b8ac0676033f6b5 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Sun, 30 Jul 2023 01:38:28 +0200 Subject: [PATCH 1/3] formats.libconfig: init Co-authored-by: ckie <25263210+ckiee@users.noreply.github.com> Signed-off-by: h7x4 --- pkgs/pkgs-lib/formats.nix | 2 + pkgs/pkgs-lib/formats/libconfig/default.nix | 121 ++++++++ .../pkgs-lib/formats/libconfig/src/Cargo.lock | 40 +++ .../pkgs-lib/formats/libconfig/src/Cargo.toml | 10 + .../formats/libconfig/src/src/main.rs | 271 ++++++++++++++++++ pkgs/pkgs-lib/formats/libconfig/update.sh | 4 + pkgs/pkgs-lib/formats/libconfig/validator.c | 21 ++ 7 files changed, 469 insertions(+) create mode 100644 pkgs/pkgs-lib/formats/libconfig/default.nix create mode 100644 pkgs/pkgs-lib/formats/libconfig/src/Cargo.lock create mode 100644 pkgs/pkgs-lib/formats/libconfig/src/Cargo.toml create mode 100644 pkgs/pkgs-lib/formats/libconfig/src/src/main.rs create mode 100755 pkgs/pkgs-lib/formats/libconfig/update.sh create mode 100644 pkgs/pkgs-lib/formats/libconfig/validator.c diff --git a/pkgs/pkgs-lib/formats.nix b/pkgs/pkgs-lib/formats.nix index 3a47d3dc849ce8c..44d4cc733078429 100644 --- a/pkgs/pkgs-lib/formats.nix +++ b/pkgs/pkgs-lib/formats.nix @@ -34,6 +34,8 @@ rec { inherit (import ./formats/java-properties/default.nix { inherit lib pkgs; }) javaProperties; + libconfig = (import ./formats/libconfig/default.nix { inherit lib pkgs; }).format; + json = {}: { type = with lib.types; let diff --git a/pkgs/pkgs-lib/formats/libconfig/default.nix b/pkgs/pkgs-lib/formats/libconfig/default.nix new file mode 100644 index 000000000000000..7433a72853533fe --- /dev/null +++ b/pkgs/pkgs-lib/formats/libconfig/default.nix @@ -0,0 +1,121 @@ +{ lib +, pkgs +}: +let + inherit (pkgs) buildPackages callPackage; + # Implementation notes: + # Libconfig spec: https://hyperrealm.github.io/libconfig/libconfig_manual.html + # + # Since libconfig does not allow setting names to start with an underscore, + # this is used as a prefix for both special types and include directives. + # + # The difference between 32bit and 64bit values became optional in libconfig + # 1.5, so we assume 64bit values for all numbers. + + libconfig-generator = buildPackages.rustPlatform.buildRustPackage { + name = "libconfig-generator"; + version = "0.1.0"; + src = ./src; + + passthru.updateScript = ./update.sh; + + cargoLock.lockFile = ./src/Cargo.lock; + }; + + libconfig-validator = buildPackages.runCommandCC "libconfig-validator" + { + buildInputs = with buildPackages; [ libconfig ]; + } + '' + mkdir -p "$out/bin" + $CC -lconfig -x c - -o "$out/bin/libconfig-validator" ${./validator.c} + ''; +in +{ + format = { generator ? libconfig-generator, validator ? libconfig-validator }: { + inherit generator; + + type = with lib.types; + let + valueType = (oneOf [ + bool + int + float + str + path + (attrsOf valueType) + (listOf valueType) + ]) // { + description = "libconfig value"; + }; + in + attrsOf valueType; + + lib = { + mkHex = value: { + _type = "hex"; + inherit value; + }; + mkOctal = value: { + _type = "octal"; + inherit value; + }; + mkFloat = value: { + _type = "float"; + inherit value; + }; + mkArray = value: { + _type = "array"; + inherit value; + }; + mkList = value: { + _type = "list"; + inherit value; + }; + }; + + generate = name: value: + callPackage + ({ + stdenvNoCC + , libconfig-generator + , libconfig-validator + , writeText + }: stdenvNoCC.mkDerivation rec { + inherit name; + + dontUnpack = true; + + json = builtins.toJSON value; + passAsFile = [ "json" ]; + + strictDeps = true; + nativeBuildInputs = [ libconfig-generator ]; + buildPhase = '' + runHook preBuild + libconfig-generator < $jsonPath > output.cfg + runHook postBuild + ''; + + doCheck = true; + nativeCheckInputs = [ libconfig-validator ]; + checkPhase = '' + runHook preCheck + libconfig-validator output.cfg + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + mv output.cfg $out + runHook postInstall + ''; + + passthru.json = writeText "${name}.json" json; + }) + { + libconfig-generator = generator; + libconfig-validator = validator; + }; + }; +} diff --git a/pkgs/pkgs-lib/formats/libconfig/src/Cargo.lock b/pkgs/pkgs-lib/formats/libconfig/src/Cargo.lock new file mode 100644 index 000000000000000..f8f921f996f92e4 --- /dev/null +++ b/pkgs/pkgs-lib/formats/libconfig/src/Cargo.lock @@ -0,0 +1,40 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "libconfig-generator" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" + +[[package]] +name = "serde_json" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +dependencies = [ + "itoa", + "ryu", + "serde", +] diff --git a/pkgs/pkgs-lib/formats/libconfig/src/Cargo.toml b/pkgs/pkgs-lib/formats/libconfig/src/Cargo.toml new file mode 100644 index 000000000000000..20ad44d221945db --- /dev/null +++ b/pkgs/pkgs-lib/formats/libconfig/src/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "libconfig-generator" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = "1.0.178" +serde_json = "1.0.104" diff --git a/pkgs/pkgs-lib/formats/libconfig/src/src/main.rs b/pkgs/pkgs-lib/formats/libconfig/src/src/main.rs new file mode 100644 index 000000000000000..4da45f647d46f1c --- /dev/null +++ b/pkgs/pkgs-lib/formats/libconfig/src/src/main.rs @@ -0,0 +1,271 @@ +use serde_json::Value; +use std::mem::discriminant; + +#[derive(Debug)] +enum LibConfigIntNumber { + Oct(i64), + Hex(i64), + Int(i64), +} + +#[derive(Debug)] +enum LibConfigValue { + Bool(bool), + Int(LibConfigIntNumber), + Float(f64), + String(String), + Array(Vec), + List(Vec), + Group(Vec, Vec<(String, LibConfigValue)>), +} + +fn validate_setting_name(key: &str) -> bool { + let first_char = key.chars().next().expect("Empty setting name"); + (first_char.is_alphabetic() || first_char == '*') + && key[1..] + .chars() + .all(|c| c.is_alphanumeric() || c == '_' || c == '*') +} + +const SPECIAL_TYPES: [&str; 5] = ["octal", "hex", "float", "list", "array"]; + +fn object_is_special_type(o: &serde_json::Map) -> Option<&str> { + o.get("_type").and_then(|x| x.as_str()).and_then(|x| { + if SPECIAL_TYPES.contains(&x) { + Some(x) + } else { + None + } + }) +} + +fn vec_is_array(v: &Vec) -> bool { + if v.is_empty() { + return true; + } + + let first_item = v.first().unwrap(); + + if match first_item { + LibConfigValue::Array(_) => true, + LibConfigValue::List(_) => true, + LibConfigValue::Group(_, _) => true, + _ => false, + } { + return false; + }; + + v[1..] + .iter() + .all(|item| discriminant(first_item) == discriminant(item)) +} + +fn json_to_libconfig(v: &Value) -> LibConfigValue { + match v { + Value::Null => panic!("Null value not allowed in libconfig"), + Value::Bool(b) => LibConfigValue::Bool(b.clone()), + Value::Number(n) => { + if n.is_i64() { + LibConfigValue::Int(LibConfigIntNumber::Int(n.as_i64().unwrap())) + } else if n.is_f64() { + LibConfigValue::Float(n.as_f64().unwrap()) + } else { + panic!("{} is not i64 or f64, cannot be represented as number in libconfig", n); + } + } + Value::String(s) => LibConfigValue::String(s.to_string()), + Value::Array(a) => { + let items = a + .iter() + .map(|item| json_to_libconfig(item)) + .collect::>(); + LibConfigValue::List(items) + } + Value::Object(o) => { + if let Some(_type) = object_is_special_type(o) { + let value = o + .get("value") + .expect(format!("Missing value for special type: {}", &_type).as_str()); + + return match _type { + "octal" => { + let str_value = value + .as_str() + .expect( + format!("Value is not a string for special type: {}", &_type) + .as_str(), + ) + .to_owned(); + + LibConfigValue::Int(LibConfigIntNumber::Oct( + i64::from_str_radix(&str_value, 8) + .expect(format!("Invalid octal value: {}", value).as_str()), + )) + } + "hex" => { + let str_value = value + .as_str() + .expect( + format!("Value is not a string for special type: {}", &_type) + .as_str(), + ) + .to_owned(); + + LibConfigValue::Int(LibConfigIntNumber::Hex( + i64::from_str_radix(&str_value[2..], 16) + .expect(format!("Invalid hex value: {}", value).as_str()), + )) + } + "float" => { + let str_value = value + .as_str() + .expect( + format!("Value is not a string for special type: {}", &_type) + .as_str(), + ) + .to_owned(); + + LibConfigValue::Float( + str_value + .parse::() + .expect(format!("Invalid float value: {}", value).as_str()), + ) + } + "list" => { + let items = value + .as_array() + .expect( + format!("Value is not an array for special type: {}", &_type) + .as_str(), + ) + .to_owned() + .iter() + .map(|item| json_to_libconfig(item)) + .collect::>(); + + LibConfigValue::List(items) + } + "array" => { + let items = value + .as_array() + .expect( + format!("Value is not an array for special type: {}", &_type) + .as_str(), + ) + .to_owned() + .iter() + .map(|item| json_to_libconfig(item)) + .collect::>(); + + if !vec_is_array(&items) { + panic!( + "This can not be an array because of its contents: {:#?}", + items + ); + } + + LibConfigValue::Array(items) + } + _ => panic!("Invalid type: {}", _type), + }; + } + + let mut items = o + .iter() + .filter(|(key, _)| key.as_str() != "_includes") + .map(|(key, value)| (key.clone(), json_to_libconfig(value))) + .collect::>(); + items.sort_by(|(a,_),(b,_)| a.partial_cmp(b).unwrap()); + + let includes = o + .get("_includes") + .map(|x| { + x.as_array() + .expect("_includes is not an array") + .iter() + .map(|x| { + x.as_str() + .expect("_includes item is not a string") + .to_owned() + }) + .collect::>() + }) + .unwrap_or(vec![]); + + for (key,_) in items.iter() { + if !validate_setting_name(key) { + panic!("Invalid setting name: {}", key); + } + } + LibConfigValue::Group(includes, items) + } + } +} + +impl ToString for LibConfigValue { + fn to_string(&self) -> String { + match self { + LibConfigValue::Bool(b) => b.to_string(), + LibConfigValue::Int(i) => match i { + LibConfigIntNumber::Oct(n) => format!("0{:o}", n), + LibConfigIntNumber::Hex(n) => format!("0x{:x}", n), + LibConfigIntNumber::Int(n) => n.to_string(), + }, + LibConfigValue::Float(n) => format!("{:?}", n), + LibConfigValue::String(s) => { + format!("\"{}\"", s.replace("\\", "\\\\").replace("\"", "\\\"")) + } + LibConfigValue::Array(a) => { + let items = a + .iter() + .map(|item| item.to_string()) + .collect::>() + .join(", "); + format!("[{}]", items) + } + LibConfigValue::List(a) => { + let items = a + .iter() + .map(|item| item.to_string()) + .collect::>() + .join(", "); + format!("({})", items) + } + LibConfigValue::Group(i, o) => { + let includes = i + .iter() + .map(|x| x.replace("\\", "\\\\").replace("\"", "\\\"")) + .map(|x| format!("@include \"{}\"", x)) + .collect::>() + .join("\n"); + let items = o + .iter() + .map(|(key, value)| format!("{}={};", key, value.to_string())) + .collect::>() + .join(""); + if includes.is_empty() { + format!("{{{}}}", items) + } else { + format!("{{\n{}\n{}}}", includes, items) + } + } + } + } +} + +fn main() { + let stdin = std::io::stdin().lock(); + let json = serde_json::Deserializer::from_reader(stdin) + .into_iter::() + .next() + .expect("Could not read content from stdin") + .expect("Could not parse JSON from stdin"); + + for (key, value) in json + .as_object() + .expect("Top level of JSON file is not an object") + { + print!("{}={};", key, json_to_libconfig(value).to_string()); + } + print!("\n\n"); +} diff --git a/pkgs/pkgs-lib/formats/libconfig/update.sh b/pkgs/pkgs-lib/formats/libconfig/update.sh new file mode 100755 index 000000000000000..ffc5ad3917f7199 --- /dev/null +++ b/pkgs/pkgs-lib/formats/libconfig/update.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env nix-shell +#!nix-shell -p cargo -i bash +cd "$(dirname "$0")" +cargo update diff --git a/pkgs/pkgs-lib/formats/libconfig/validator.c b/pkgs/pkgs-lib/formats/libconfig/validator.c new file mode 100644 index 000000000000000..738be0b774b5a0f --- /dev/null +++ b/pkgs/pkgs-lib/formats/libconfig/validator.c @@ -0,0 +1,21 @@ +// Copyright (C) 2005-2023 Mark A Lindner, ckie +// SPDX-License-Identifier: LGPL-2.1-or-later +#include +#include +int main(int argc, char **argv) +{ + config_t cfg; + config_init(&cfg); + if (argc != 2) + { + fprintf(stderr, "USAGE: validator "); + } + if(! config_read_file(&cfg, argv[1])) + { + fprintf(stderr, "[libconfig] %s:%d - %s\n", config_error_file(&cfg), + config_error_line(&cfg), config_error_text(&cfg)); + config_destroy(&cfg); + return 1; + } + printf("[libconfig] validation ok\n"); +} \ No newline at end of file From 18ca8b21e2918340f77d868e509c59df90f73e45 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Sun, 30 Jul 2023 01:45:03 +0200 Subject: [PATCH 2/3] formats.libconfig: add tests Co-authored-by: ckie <25263210+ckiee@users.noreply.github.com> Signed-off-by: h7x4 --- .../libconfig/test/comprehensive/default.nix | 76 +++++++++++++++++++ .../libconfig/test/comprehensive/expected.txt | 6 ++ .../formats/libconfig/test/default.nix | 4 + pkgs/pkgs-lib/tests/default.nix | 1 + 4 files changed, 87 insertions(+) create mode 100644 pkgs/pkgs-lib/formats/libconfig/test/comprehensive/default.nix create mode 100644 pkgs/pkgs-lib/formats/libconfig/test/comprehensive/expected.txt create mode 100644 pkgs/pkgs-lib/formats/libconfig/test/default.nix diff --git a/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/default.nix b/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/default.nix new file mode 100644 index 000000000000000..3715e2e840d2685 --- /dev/null +++ b/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/default.nix @@ -0,0 +1,76 @@ +{ lib, formats, stdenvNoCC, writeText, ... }: +let + libconfig = formats.libconfig { }; + + include_expr = { + val = 1; + }; + + include_file = (writeText "libconfig-test-include" '' + val=1; + '').overrideAttrs { + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + }; + + expression = { + simple_top_level_attr = "1.0"; + nested.attrset.has.a.integer.value = 100; + some_floaty = 29.95; + ## Same syntax here on these two, but they should get serialized differently: + # > A list may have zero or more elements, each of which can be a scalar value, an array, a group, or another list. + list1d = libconfig.lib.mkList [ 1 "mixed!" 5 2 ]; + # You might also omit the mkList, as a list will be a list (in contrast to an array) by default. + list2d = [ 1 [ 1 1.2 "foo" ] [ "bar" 1.2 1 ] ]; + # > An array may have zero or more elements, but the elements must all be scalar values of the same type. + array1d = libconfig.lib.mkArray [ 1 5 2 ]; + array2d = [ + (libconfig.lib.mkArray [ 1 2 ]) + (libconfig.lib.mkArray [ 2 1 ]) + ]; + nasty_string = "\"@\n\\\t^*\b\f\n\0\";'''$"; + + weirderTypes = { + _includes = [ include_file ]; + pi = 3.141592654; + bigint = 9223372036854775807; + hex = libconfig.lib.mkHex "0x1FC3"; + octal = libconfig.lib.mkOctal "0027"; + float = libconfig.lib.mkFloat "1.2E-3"; + array_of_ints = libconfig.lib.mkArray [ + (libconfig.lib.mkOctal "0732") + (libconfig.lib.mkHex "0xA3") + 1234 + ]; + list_of_weird_types = [ + 3.141592654 + 9223372036854775807 + (libconfig.lib.mkHex "0x1FC3") + (libconfig.lib.mkOctal "0027") + (libconfig.lib.mkFloat "1.2E-32") + (libconfig.lib.mkFloat "1") + ]; + }; + }; + + libconfig-test-cfg = libconfig.generate "libconfig-test.cfg" expression; +in + stdenvNoCC.mkDerivation { + name = "pkgs.formats.libconfig-test-comprehensive"; + + dontUnpack = true; + dontBuild = true; + + doCheck = true; + checkPhase = '' + diff -U3 ${./expected.txt} ${libconfig-test-cfg} + ''; + + installPhase = '' + mkdir $out + cp ${./expected.txt} $out/expected.txt + cp ${libconfig-test-cfg} $out/libconfig-test.cfg + cp ${libconfig-test-cfg.passthru.json} $out/libconfig-test.json + ''; + } + diff --git a/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/expected.txt b/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/expected.txt new file mode 100644 index 000000000000000..3e9b1af821f1825 --- /dev/null +++ b/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/expected.txt @@ -0,0 +1,6 @@ +array1d=[1, 5, 2];array2d=([1, 2], [2, 1]);list1d=(1, "mixed!", 5, 2);list2d=(1, (1, 1.2, "foo"), ("bar", 1.2, 1));nasty_string="\"@ +\\ ^*bf +0\";'''$";nested={attrset={has={a={integer={value=100;};};};};};simple_top_level_attr="1.0";some_floaty=29.95;weirderTypes={ +@include "/nix/store/jdz5yhzbbj4j77yrr7l20x1cs4kbwkj2-libconfig-test-include" +array_of_ints=[0732, 0xa3, 1234];bigint=9223372036854775807;float=0.0012;hex=0x1fc3;list_of_weird_types=(3.141592654, 9223372036854775807, 0x1fc3, 027, 1.2e-32, 1.0);octal=027;pi=3.141592654;}; + diff --git a/pkgs/pkgs-lib/formats/libconfig/test/default.nix b/pkgs/pkgs-lib/formats/libconfig/test/default.nix new file mode 100644 index 000000000000000..6cd03fe4854f59c --- /dev/null +++ b/pkgs/pkgs-lib/formats/libconfig/test/default.nix @@ -0,0 +1,4 @@ +{ pkgs, ... }: +{ + comprehensive = pkgs.callPackage ./comprehensive { }; +} diff --git a/pkgs/pkgs-lib/tests/default.nix b/pkgs/pkgs-lib/tests/default.nix index ae91e15aa9efada..289780f57650a9a 100644 --- a/pkgs/pkgs-lib/tests/default.nix +++ b/pkgs/pkgs-lib/tests/default.nix @@ -17,6 +17,7 @@ let jdk11 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk11_headless; }; jdk17 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk17_headless; }; }; + libconfig = recurseIntoAttrs (import ../formats/libconfig/test { inherit pkgs; }); }; flatten = prefix: as: From 5707d01df2f08e0be3e70898f7648fc4ba341316 Mon Sep 17 00:00:00 2001 From: ckie Date: Mon, 16 Oct 2023 03:29:45 +0300 Subject: [PATCH 3/3] pkgs-lib.formats: add note about missing `pkgs` quirk --- pkgs/pkgs-lib/formats.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/pkgs-lib/formats.nix b/pkgs/pkgs-lib/formats.nix index 44d4cc733078429..3cbda3a7ebdd96f 100644 --- a/pkgs/pkgs-lib/formats.nix +++ b/pkgs/pkgs-lib/formats.nix @@ -28,6 +28,11 @@ rec { generate = ...; }); + + Please note that `pkgs` may not always be available for use due to the split + options doc build introduced in fc614c37c653, so lazy evaluation of only the + 'type' field is required. + */