feat(cli): agama profile autoyast command (#1279)

## Problem

Call `agama profile download` do for autoyast beside downloading also
convert to json which is confusing to use e.g. `agama profile download
ftp://test.com/autoyast.xml > profile.json`


## Solution

For downloading use `agama download` and for special autoyast processing
regarding URL and also result use `agama profile autoyast`

autoinstallation/README.md

@@ -104,12 +104,12 @@ this scenario, it is expected to use the CLI to interact with Agama. In addition
 any other tool available in the installation media. What's more, when using the Live ISO, you could
 install your own tools.
 
-Below there is a minimal working example to install ALP Dolomite:
+Below there is a minimal working example to install Tumbleweed:
 
 ```sh
 set -ex
 
-/usr/bin/agama config set software.product=ALP-Dolomite
+/usr/bin/agama config set product.id=Tumbleweed
 /usr/bin/agama config set user.userName=joe user.password=doe
 /usr/bin/agama install
 ```
@@ -133,9 +133,9 @@ internal network.
 ```sh
 set -ex
 
-/usr/bin/agama profile download ftp://my.server/tricky_hardware_setup.sh
+/usr/bin/agama download ftp://my.server/tricky_hardware_setup.sh > tricky_hardware_setup.sh
 sh tricky_hardware_setup.sh
-/usr/bin/agama config set software.product=Tumbleweed
+/usr/bin/agama config set product.id=Tumbleweed
 /usr/bin/agama config set user.userName=joe user.password=doe
 /usr/bin/agama install
 ```
@@ -147,13 +147,11 @@ Jsonnet may be unable to handle all of the profile changes that users wish to ma
 ```
 set -ex
 
-/usr/bin/agama profile download ftp://my.server/profile.json
+/usr/bin/agama download ftp://my.server/profile.json > /root/profile.json
 
 # modify profile.json here
 
-/usr/bin/agama profile validate profile.json
-/usr/bin/agama config load profile.json
-
+/usr/bin/agama profile import file:///root/profile.json
 /usr/bin/agama install
 ```
 
@@ -169,7 +167,7 @@ Agama and before installing RPMs, such as changing the fstab and mount an extra
 ```sh
 set -ex
 
-/usr/bin/agama config set software.product=Tumbleweed
+/usr/bin/agama config set product.id=Tumbleweed
 /usr/bin/agama config set user.userName=joe user.password=doe
 
 /usr/bin/agama install --until partitioning # install till the partitioning step
@@ -191,9 +189,9 @@ software for internal network, then it must be modified before umount.
 ```sh
 set -ex
 
-/usr/bin/agama profile download ftp://my.server/velociraptor.config
+/usr/bin/agama download ftp://my.server/velociraptor.config
 
-/usr/bin/agama config set software.product=Tumbleweed
+/usr/bin/agama config set product.id=Tumbleweed
 /usr/bin/agama config set user.userName=joe user.password=doe
 
 /usr/bin/agama install --until deploy # do partitioning, rpm installation and configuration step
@@ -218,7 +216,7 @@ some kernel tuning or adding some remote storage that needs to be mounted during
 ```sh
 set -ex
 
-/usr/bin/agama config set software.product=Tumbleweed
+/usr/bin/agama config set product.id=Tumbleweed
 /usr/bin/agama config set user.userName=joe user.password=doe
 
 /usr/bin/agama install --until deploy # do partitioning, rpm installation and configuration step

rust/agama-cli/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 clap = { version = "4.1.4", features = ["derive", "wrap_help"] }
+curl = { version = "0.4.44", features = ["protocol-ftp"] }
 agama-lib = { path="../agama-lib" }
 agama-settings = { path="../agama-settings" }
 serde = { version = "1.0.152" }
@@ -28,6 +29,7 @@ async-trait = "0.1.77"
 reqwest = { version = "0.11", features = ["json"] }
 home = "0.5.9"
 rpassword = "7.3.1"
+url = "2.5.0"
 
 [[bin]]
 name = "agama"

rust/agama-cli/src/commands.rs

@@ -72,4 +72,16 @@ pub enum Commands {
     /// not affect the root user.
     #[command(subcommand)]
     Auth(AuthCommands),
+
+    /// Download file from given URL
+    ///
+    /// The purpose of this command is to download files using AutoYaST supported schemas (e.g. device:// or relurl://).
+    /// It can be used to download additional scripts, configuration files and so on.
+    /// You can use it for downloading Agama autoinstallation profiles. However, unless you need additional processing,
+    /// the "agama profile import" is recommended.
+    /// If you want to convert an AutoYaST profile, use "agama profile autoyast".
+    Download {
+        /// URL pointing to file for download
+        url: String,
+    },
 }

rust/agama-cli/src/config.rs

@@ -35,10 +35,15 @@ pub enum ConfigCommands {
     ///
     /// It is possible that many configuration settings do not have a value. Those settings
     /// are not included in the output.
+    ///
+    /// The output of command can be used as file content for `agama config load`.
     Show,
 
     /// Loads the configuration from a JSON file.
-    Load { path: String },
+    Load {
+        /// Local path to file with configuration. For schema see /usr/share/agama-cli/profile.json.schema
+        path: String,
+    },
 }
 
 pub enum ConfigAction {

rust/agama-cli/src/main.rs

@@ -144,6 +144,7 @@ async fn run_command(cli: Cli) -> anyhow::Result<()> {
         Commands::Questions(subcommand) => run_questions_cmd(subcommand).await,
         Commands::Logs(subcommand) => run_logs_cmd(subcommand).await,
         Commands::Auth(subcommand) => run_auth_cmd(subcommand).await,
+        Commands::Download { url } => crate::profile::download(&url, std::io::stdout()),
     }
 }
 

rust/agama-cli/src/profile.rs

@@ -1,6 +1,7 @@
-use agama_lib::profile::{ProfileEvaluator, ProfileReader, ProfileValidator, ValidationResult};
+use agama_lib::profile::{AutoyastProfile, ProfileEvaluator, ProfileValidator, ValidationResult};
 use anyhow::Context;
 use clap::Subcommand;
+use curl::easy::Easy;
 use std::os::unix::process::CommandExt;
 use std::{
     fs::File,
@@ -9,32 +10,59 @@ use std::{
     process::Command,
 };
 use tempfile::TempDir;
+use url::Url;
 
 #[derive(Subcommand, Debug)]
 pub enum ProfileCommands {
-    /// Download the profile from a given location
-    Download { url: String },
+    /// Download the autoyast profile and print resulting json
+    Autoyast {
+        /// AutoYaST profile's URL. Any AutoYaST scheme, ERB and rules/classes are supported.
+        /// all schemas that autoyast supports.
+        url: String,
+    },
 
     /// Validate a profile using JSON Schema
-    Validate { path: String },
+    ///
+    /// Schema is available at /usr/share/agama-cli/profile.schema.json
+    Validate {
+        /// Local path to json file
+        path: String,
+    },
 
     /// Evaluate a profile, injecting the hardware information from D-Bus
-    Evaluate { path: String },
+    ///
+    /// For an example of Jsonnet-based profile, see
+    /// https://github.com/openSUSE/agama/blob/master/rust/agama-lib/share/examples/profile.jsonnet
+    Evaluate {
+        /// Path to jsonnet file.
+        path: String,
+    },
 
     /// Process autoinstallation profile and loads it into agama
     ///
     /// This is top level command that do all autoinstallation processing beside starting
     /// installation. Unless there is a need to inject additional commands between processing
     /// use this command instead of set of underlying commands.
-    /// Optional dir argument is location where profile is processed. Useful for debugging
-    /// if something goes wrong.
-    Import { url: String, dir: Option<PathBuf> },
+    Import {
+        /// Profile's URL. Supports the same schemas than te "download" command plus
+        /// AutoYaST specific ones. Supported files are json, jsonnet, sh for Agama profiles and ERB, XML, and rules/classes directories
+        /// for AutoYaST support.
+        url: String,
+        /// Specific directory where all processing happens. By default it uses a temporary directory
+        dir: Option<PathBuf>,
+    },
 }
 
-fn download(url: &str, mut out_fd: impl Write) -> anyhow::Result<()> {
-    let reader = ProfileReader::new(url)?;
-    let contents = reader.read()?;
-    out_fd.write_all(contents.as_bytes())?;
+pub fn download(url: &str, mut out_fd: impl Write) -> anyhow::Result<()> {
+    let mut handle = Easy::new();
+    handle.url(url)?;
+
+    let mut transfer = handle.transfer();
+    transfer.write_function(|buf|
+        // unwrap here is ok, as we want to kill download if we failed to write content
+        Ok(out_fd.write(buf).unwrap()))?;
+    transfer.perform()?;
+
     Ok(())
 }
 
@@ -66,20 +94,28 @@ fn evaluate(path: String) -> anyhow::Result<()> {
     Ok(())
 }
 
-async fn import(url: String, dir: Option<PathBuf>) -> anyhow::Result<()> {
+async fn import(url_string: String, dir: Option<PathBuf>) -> anyhow::Result<()> {
+    let url = Url::parse(&url_string)?;
     let tmpdir = TempDir::new()?; // TODO: create it only if dir is not passed
-    let output_file = if url.ends_with(".sh") {
+    let path = url.path();
+    let output_file = if path.ends_with(".sh") {
         "profile.sh"
-    } else if url.ends_with(".jsonnet") {
+    } else if path.ends_with(".jsonnet") {
         "profile.jsonnet"
     } else {
         "profile.json"
     };
     let output_dir = dir.unwrap_or_else(|| tmpdir.into_path());
     let mut output_path = output_dir.join(output_file);
     let output_fd = File::create(output_path.clone())?;
-    //download profile
-    download(&url, output_fd)?;
+    if path.ends_with(".xml") || path.ends_with(".erb") || path.ends_with('/') {
+        // autoyast specific download and convert to json
+        AutoyastProfile::new(&url)?.read_into(output_fd)?;
+    } else {
+        // just download profile
+        download(&url_string, output_fd)?;
+    }
+
     // exec shell scripts
     if output_file.ends_with(".sh") {
         let err = Command::new("bash")
@@ -117,9 +153,16 @@ async fn import(url: String, dir: Option<PathBuf>) -> anyhow::Result<()> {
     Ok(())
 }
 
+fn autoyast(url_string: String) -> anyhow::Result<()> {
+    let url = Url::parse(&url_string)?;
+    let reader = AutoyastProfile::new(&url)?;
+    reader.read_into(std::io::stdout())?;
+    Ok(())
+}
+
 pub async fn run(subcommand: ProfileCommands) -> anyhow::Result<()> {
     match subcommand {
-        ProfileCommands::Download { url } => download(&url, std::io::stdout()),
+        ProfileCommands::Autoyast { url } => autoyast(url),
         ProfileCommands::Validate { path } => validate(path),
         ProfileCommands::Evaluate { path } => evaluate(path),
         ProfileCommands::Import { url, dir } => import(url, dir).await,

rust/agama-lib/Cargo.toml

@@ -10,7 +10,6 @@ agama-settings = { path="../agama-settings" }
 anyhow = "1.0"
 async-trait = "0.1.77"
 cidr = { version = "0.2.2", features = ["serde"] }
-curl = { version = "0.4.44", features = ["protocol-ftp"] }
 futures-util = "0.3.29"
 jsonschema = { version = "0.16.1", default-features = false }
 log = "0.4"
@@ -25,3 +24,5 @@ tokio-stream = "0.1.14"
 url = "2.5.0"
 utoipa = "4.2.0"
 zbus = { version = "3", default-features = false, features = ["tokio"] }
+# Needed to define curl error in profile errors
+curl = { version = "0.4.44", features = ["protocol-ftp"] }

rust/agama-lib/src/profile.rs

@@ -1,49 +1,34 @@
 use crate::error::ProfileError;
 use anyhow::Context;
-use curl::easy::Easy;
 use jsonschema::JSONSchema;
 use log::info;
 use serde_json;
 use std::{fs, io::Write, path::Path, process::Command};
 use tempfile::{tempdir, TempDir};
 use url::Url;
 
-/// Downloads a profile for a given location.
-pub struct ProfileReader {
+/// Downloads and converts autoyast profile.
+pub struct AutoyastProfile {
     url: Url,
 }
 
-impl ProfileReader {
-    pub fn new(url: &str) -> anyhow::Result<Self> {
-        let url = Url::parse(url)?;
-        Ok(Self { url })
+impl AutoyastProfile {
+    pub fn new(url: &Url) -> anyhow::Result<Self> {
+        Ok(Self { url: url.clone() })
     }
 
-    pub fn read(&self) -> anyhow::Result<String> {
+    pub fn read_into(&self, mut out_fd: impl Write) -> anyhow::Result<()> {
         let path = self.url.path();
         if path.ends_with(".xml") || path.ends_with(".erb") || path.ends_with('/') {
-            self.read_from_autoyast()
+            let content = self.read_from_autoyast()?;
+            out_fd.write_all(content.as_bytes())?;
+            Ok(())
         } else {
-            self.read_from_url()
+            let msg = format!("Unsupported AutoYaST format at {}", self.url);
+            Err(anyhow::Error::msg(msg))
         }
     }
 
-    fn read_from_url(&self) -> anyhow::Result<String> {
-        let mut buf = Vec::new();
-        {
-            let mut handle = Easy::new();
-            handle.url(self.url.as_str())?;
-
-            let mut transfer = handle.transfer();
-            transfer.write_function(|data| {
-                buf.extend(data);
-                Ok(data.len())
-            })?;
-            transfer.perform().unwrap();
-        }
-        Ok(String::from_utf8(buf)?)
-    }
-
     fn read_from_autoyast(&self) -> anyhow::Result<String> {
         const TMP_DIR_PREFIX: &str = "autoyast";
         const AUTOINST_JSON: &str = "autoinst.json";

rust/package/agama.changes

@@ -1,3 +1,11 @@
+-------------------------------------------------------------------
+Mon Jun  3 07:49:16 UTC 2024 - Josef Reidinger <[email protected]>
+
+- CLI: Add new commands "agama download" and
+  "agama profile autoyast" and remove "agama profile download" to
+  separate common curl-like download and autoyast specific one
+  which do conversion to json (gh#openSUSE/agama#1279)
+
 -------------------------------------------------------------------
 Wed May 29 12:15:37 UTC 2024 - Josef Reidinger <[email protected]>