From c5d729540a709fce6207017e90dfe68d86c74562 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 14 Nov 2022 13:33:28 +1100 Subject: [PATCH 01/45] config: swap out envy for config dependency --- htsget-config/Cargo.toml | 2 +- htsget-config/src/config.rs | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index ab93caf3a..0e49f4c25 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -12,6 +12,6 @@ default = ["s3-storage"] serde = { version = "1.0", features = ["derive"] } serde_regex = "1.1" regex = "1.6" -envy = "0.4" +config = "0.13" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] } \ No newline at end of file diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index 0a0ee5a89..ee2431f19 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -49,7 +49,7 @@ The next variables are used to configure the info for the service-info endpoints * HTSGET_ENVIRONMENT: The environment in which the service is running. Default: "None". "#; -const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET_"; +const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET"; fn default_localstorage_addr() -> SocketAddr { "127.0.0.1:8081".parse().expect("expected valid address") @@ -175,14 +175,20 @@ impl Config { /// Read the environment variables into a Config struct. #[instrument] pub fn from_env() -> io::Result { - let config = envy::prefixed(ENVIRONMENT_VARIABLE_PREFIX) - .from_env() + let config = config::Config::builder() + .add_source(config::Environment::with_prefix( + ENVIRONMENT_VARIABLE_PREFIX, + )) + .build() .map_err(|err| { io::Error::new( ErrorKind::Other, format!("config not properly set: {}", err), ) - }); + })? + .try_deserialize::() + .map_err(|err| io::Error::new(ErrorKind::Other, format!("failed to parse config: {}", err))); + info!(config = ?config, "config created from environment variables"); config } From d9f2ad7d987442207c4d0fcafd45629ce2023cab Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 15 Nov 2022 13:35:54 +1100 Subject: [PATCH 02/45] config: add config file from command line or env option --- htsget-config/Cargo.toml | 1 + htsget-config/src/config.rs | 36 ++++++++++++++++++++++++---------- htsget-http-actix/src/main.rs | 12 ++---------- htsget-http-lambda/src/main.rs | 2 +- 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 0e49f4c25..418a4831a 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -13,5 +13,6 @@ serde = { version = "1.0", features = ["derive"] } serde_regex = "1.1" regex = "1.6" config = "0.13" +clap = { version = "4.0", features = ["derive", "env"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] } \ No newline at end of file diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index ee2431f19..db0b85b44 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -3,6 +3,8 @@ use std::io::ErrorKind; use std::net::SocketAddr; use std::path::PathBuf; +use clap::Parser; +use config::File; use serde::Deserialize; use tracing::info; use tracing::instrument; @@ -14,7 +16,7 @@ use crate::regex_resolver::RegexResolver; /// Represents a usage string for htsget-rs. pub const USAGE: &str = r#" -The HtsGet server executables don't use command line arguments, but there are some environment variables that can be set to configure them: +Available environment variables: * HTSGET_PATH: The path to the directory where the server should be started. Default: "data". Unused if HTSGET_STORAGE_TYPE is "AwsS3Storage". * HTSGET_REGEX: The regular expression that should match an ID. Default: ".*". For more information about the regex options look in the documentation of the regex crate(https://docs.rs/regex/). @@ -71,6 +73,14 @@ fn default_path() -> PathBuf { PathBuf::from("data") } +/// The command line arguments allowed for the htsget-rs executables. +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = USAGE)] +pub struct Args { + #[arg(short, long, env = "HTSGET_CONFIG")] + config: PathBuf, +} + /// Specify the storage type to use. #[derive(Deserialize, Debug, Clone, PartialEq, Eq)] #[non_exhaustive] @@ -172,10 +182,16 @@ impl Default for Config { } impl Config { + /// Parse the command line arguments + pub fn parse_args() -> PathBuf { + Args::parse().config + } + /// Read the environment variables into a Config struct. #[instrument] - pub fn from_env() -> io::Result { + pub fn from_env(config: PathBuf) -> io::Result { let config = config::Config::builder() + .add_source(File::from(config)) .add_source(config::Environment::with_prefix( ENVIRONMENT_VARIABLE_PREFIX, )) @@ -218,7 +234,7 @@ mod tests { #[test] fn config_addr() { std::env::set_var("HTSGET_TICKET_SERVER_ADDR", "127.0.0.1:8081"); - let config = Config::from_env().unwrap(); + let config = Config::from_env(PathBuf::default()).unwrap(); assert_eq!( config.ticket_server_config.ticket_server_addr, "127.0.0.1:8081".parse().unwrap() @@ -231,7 +247,7 @@ mod tests { "HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGIN", "http://localhost:8080", ); - let config = Config::from_env().unwrap(); + let config = Config::from_env(PathBuf::default()).unwrap(); assert_eq!( config.ticket_server_config.ticket_server_cors_allow_origin, "http://localhost:8080" @@ -244,7 +260,7 @@ mod tests { "HTSGET_DATA_SERVER_CORS_ALLOW_ORIGIN", "http://localhost:8080", ); - let config = Config::from_env().unwrap(); + let config = Config::from_env(PathBuf::default()).unwrap(); assert_eq!( config.data_server_config.data_server_cors_allow_origin, "http://localhost:8080" @@ -254,7 +270,7 @@ mod tests { #[test] fn config_ticket_server_addr() { std::env::set_var("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082"); - let config = Config::from_env().unwrap(); + let config = Config::from_env(PathBuf::default()).unwrap(); assert_eq!( config.data_server_config.data_server_addr, "127.0.0.1:8082".parse().unwrap() @@ -264,21 +280,21 @@ mod tests { #[test] fn config_regex() { std::env::set_var("HTSGET_REGEX", ".+"); - let config = Config::from_env().unwrap(); + let config = Config::from_env(PathBuf::default()).unwrap(); assert_eq!(config.resolver.regex.to_string(), ".+"); } #[test] fn config_substitution_string() { std::env::set_var("HTSGET_SUBSTITUTION_STRING", "$0-test"); - let config = Config::from_env().unwrap(); + let config = Config::from_env(PathBuf::default()).unwrap(); assert_eq!(config.resolver.substitution_string, "$0-test"); } #[test] fn config_service_info_id() { std::env::set_var("HTSGET_ID", "id"); - let config = Config::from_env().unwrap(); + let config = Config::from_env(PathBuf::default()).unwrap(); assert_eq!(config.ticket_server_config.service_info.id.unwrap(), "id"); } @@ -286,7 +302,7 @@ mod tests { #[test] fn config_storage_type() { std::env::set_var("HTSGET_STORAGE_TYPE", "AwsS3Storage"); - let config = Config::from_env().unwrap(); + let config = Config::from_env(PathBuf::default()).unwrap(); assert_eq!(config.storage_type, StorageType::AwsS3Storage); } } diff --git a/htsget-http-actix/src/main.rs b/htsget-http-actix/src/main.rs index c17503f16..3c4e7ef1a 100644 --- a/htsget-http-actix/src/main.rs +++ b/htsget-http-actix/src/main.rs @@ -1,24 +1,16 @@ -use std::env::args; use std::io::{Error, ErrorKind}; use tokio::select; use htsget_http_actix::run_server; -use htsget_http_actix::{Config, StorageType, USAGE}; +use htsget_http_actix::{Config, StorageType}; use htsget_search::htsget::from_storage::HtsGetFromStorage; use htsget_search::storage::data_server::HttpTicketFormatter; #[actix_web::main] async fn main() -> std::io::Result<()> { Config::setup_tracing()?; - - if args().len() > 1 { - // Show help if command line options are provided - println!("{}", USAGE); - return Ok(()); - } - - let config = Config::from_env()?; + let config = Config::from_env(Config::parse_args())?; match config.storage_type { StorageType::LocalStorage => local_storage_server(config).await, diff --git a/htsget-http-lambda/src/main.rs b/htsget-http-lambda/src/main.rs index dd4529ad0..48932c77e 100644 --- a/htsget-http-lambda/src/main.rs +++ b/htsget-http-lambda/src/main.rs @@ -12,7 +12,7 @@ use htsget_search::storage::local::LocalStorage; #[tokio::main] async fn main() -> Result<(), Error> { Config::setup_tracing()?; - let config = Config::from_env()?; + let config = Config::from_env(Config::parse_args())?; match config.storage_type { StorageType::LocalStorage => local_storage_server(config).await, From 225eb73d0c488ced1d83d62e7beeee64cb4a11da Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 16 Nov 2022 10:03:24 +1100 Subject: [PATCH 03/45] config: add separate config for local server and s3 storage --- htsget-config/config.toml | 0 htsget-config/src/config.rs | 54 +++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 htsget-config/config.toml diff --git a/htsget-config/config.toml b/htsget-config/config.toml new file mode 100644 index 000000000..e69de29bb diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index db0b85b44..bdc764cf7 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -73,6 +73,10 @@ fn default_path() -> PathBuf { PathBuf::from("data") } +fn default_serve_at() -> PathBuf { + PathBuf::from("/data") +} + /// The command line arguments allowed for the htsget-rs executables. #[derive(Parser, Debug)] #[command(author, version, about, long_about = USAGE)] @@ -117,6 +121,40 @@ pub struct TicketServerConfig { pub ticket_server_cors_allow_origin: String, } +/// Configuration for the htsget server. +#[derive(Deserialize, Debug, Clone)] +#[serde(default)] +pub struct LocalDataServer { + pub path: PathBuf, + pub serve_at: PathBuf, + pub addr: SocketAddr, + pub key: Option, + pub cert: Option, + pub cors_allow_credentials: bool, + pub cors_allow_origin: String, +} + +impl Default for LocalDataServer { + fn default() -> Self { + Self { + path: default_path(), + serve_at: default_serve_at(), + addr: default_localstorage_addr(), + key: None, + cert: None, + cors_allow_credentials: false, + cors_allow_origin: default_data_server_origin(), + } + } +} + +/// Configuration for the htsget server. +#[derive(Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct AwsS3DataServer { + pub bucket: String, +} + /// Configuration for the htsget server. #[derive(Deserialize, Debug, Clone)] #[serde(default)] @@ -234,7 +272,7 @@ mod tests { #[test] fn config_addr() { std::env::set_var("HTSGET_TICKET_SERVER_ADDR", "127.0.0.1:8081"); - let config = Config::from_env(PathBuf::default()).unwrap(); + let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); assert_eq!( config.ticket_server_config.ticket_server_addr, "127.0.0.1:8081".parse().unwrap() @@ -247,7 +285,7 @@ mod tests { "HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGIN", "http://localhost:8080", ); - let config = Config::from_env(PathBuf::default()).unwrap(); + let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); assert_eq!( config.ticket_server_config.ticket_server_cors_allow_origin, "http://localhost:8080" @@ -260,7 +298,7 @@ mod tests { "HTSGET_DATA_SERVER_CORS_ALLOW_ORIGIN", "http://localhost:8080", ); - let config = Config::from_env(PathBuf::default()).unwrap(); + let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); assert_eq!( config.data_server_config.data_server_cors_allow_origin, "http://localhost:8080" @@ -270,7 +308,7 @@ mod tests { #[test] fn config_ticket_server_addr() { std::env::set_var("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082"); - let config = Config::from_env(PathBuf::default()).unwrap(); + let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); assert_eq!( config.data_server_config.data_server_addr, "127.0.0.1:8082".parse().unwrap() @@ -280,21 +318,21 @@ mod tests { #[test] fn config_regex() { std::env::set_var("HTSGET_REGEX", ".+"); - let config = Config::from_env(PathBuf::default()).unwrap(); + let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); assert_eq!(config.resolver.regex.to_string(), ".+"); } #[test] fn config_substitution_string() { std::env::set_var("HTSGET_SUBSTITUTION_STRING", "$0-test"); - let config = Config::from_env(PathBuf::default()).unwrap(); + let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); assert_eq!(config.resolver.substitution_string, "$0-test"); } #[test] fn config_service_info_id() { std::env::set_var("HTSGET_ID", "id"); - let config = Config::from_env(PathBuf::default()).unwrap(); + let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); assert_eq!(config.ticket_server_config.service_info.id.unwrap(), "id"); } @@ -302,7 +340,7 @@ mod tests { #[test] fn config_storage_type() { std::env::set_var("HTSGET_STORAGE_TYPE", "AwsS3Storage"); - let config = Config::from_env(PathBuf::default()).unwrap(); + let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); assert_eq!(config.storage_type, StorageType::AwsS3Storage); } } From 2bbfed9ed05d2ec929ddb2ca3fff819e1f3e99a7 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 17 Nov 2022 09:08:18 +1100 Subject: [PATCH 04/45] config: move fields, tags, no tags, query, and interval to config --- htsget-config/Cargo.toml | 1 + htsget-config/src/config.rs | 3 + htsget-config/src/lib.rs | 232 +++++++++++++++++++++ htsget-config/src/regex_resolver.rs | 62 +++++- htsget-http-core/src/lib.rs | 10 +- htsget-http-core/src/post_request.rs | 4 +- htsget-http-core/src/query_builder.rs | 8 +- htsget-http-core/src/service_info.rs | 3 +- htsget-http-lambda/src/lib.rs | 3 +- htsget-search/benches/search_benchmarks.rs | 10 +- htsget-search/src/htsget/bam_search.rs | 13 +- htsget-search/src/htsget/bcf_search.rs | 2 +- htsget-search/src/htsget/cram_search.rs | 35 ++-- htsget-search/src/htsget/from_storage.rs | 3 +- htsget-search/src/htsget/mod.rs | 219 +------------------ htsget-search/src/htsget/search.rs | 97 +++------ htsget-search/src/htsget/vcf_search.rs | 2 +- htsget-search/src/storage/aws.rs | 102 +++++---- htsget-search/src/storage/local.rs | 92 ++++---- htsget-search/src/storage/mod.rs | 24 +-- htsget-test-utils/src/server_tests.rs | 12 +- 21 files changed, 486 insertions(+), 451 deletions(-) diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 418a4831a..214a75f32 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -9,6 +9,7 @@ s3-storage = [] default = ["s3-storage"] [dependencies] +noodles = { version = "0.29", features = ["core"] } serde = { version = "1.0", features = ["derive"] } serde_regex = "1.1" regex = "1.6" diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index bdc764cf7..a0acddbc4 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -153,6 +153,7 @@ impl Default for LocalDataServer { #[serde(default)] pub struct AwsS3DataServer { pub bucket: String, + pub regex_resolvers: Vec, } /// Configuration for the htsget server. @@ -164,6 +165,7 @@ pub struct DataServerConfig { pub data_server_cert: Option, pub data_server_cors_allow_credentials: bool, pub data_server_cors_allow_origin: String, + pub regex_resolvers: Vec, } /// Configuration of the service info. @@ -201,6 +203,7 @@ impl Default for DataServerConfig { data_server_cert: None, data_server_cors_allow_credentials: false, data_server_cors_allow_origin: default_data_server_origin(), + regex_resolvers: vec![RegexResolver::default()], } } } diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 0869818f4..4acd5441e 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -1,2 +1,234 @@ +extern crate core; + +use noodles::core::region::Interval as NoodlesInterval; +use noodles::core::Position; +use serde::{Deserialize, Serialize}; +use std::fmt::Formatter; +use std::io::ErrorKind::Other; +use std::{fmt, io}; +use tracing::instrument; + pub mod config; pub mod regex_resolver; + +/// An enumeration with all the possible formats. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum Format { + Bam, + Cram, + Vcf, + Bcf, +} + +impl Format { + pub fn fmt_file(&self, id: &str) -> String { + match self { + Format::Bam => format!("{}.bam", id), + Format::Cram => format!("{}.cram", id), + Format::Vcf => format!("{}.vcf.gz", id), + Format::Bcf => format!("{}.bcf", id), + } + } + + pub fn fmt_index(&self, id: &str) -> String { + match self { + Format::Bam => format!("{}.bam.bai", id), + Format::Cram => format!("{}.cram.crai", id), + Format::Vcf => format!("{}.vcf.gz.tbi", id), + Format::Bcf => format!("{}.bcf.csi", id), + } + } + + pub fn fmt_gzi(&self, id: &str) -> io::Result { + match self { + Format::Bam => Ok(format!("{}.bam.gzi", id)), + Format::Cram => Err(io::Error::new( + Other, + "CRAM does not support GZI".to_string(), + )), + Format::Vcf => Ok(format!("{}.vcf.gz.gzi", id)), + Format::Bcf => Ok(format!("{}.bcf.gzi", id)), + } + } +} + +impl From for String { + fn from(format: Format) -> Self { + format.to_string() + } +} + +impl fmt::Display for Format { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Format::Bam => write!(f, "BAM"), + Format::Cram => write!(f, "CRAM"), + Format::Vcf => write!(f, "VCF"), + Format::Bcf => write!(f, "BCF"), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Class { + Header, + Body, +} + +/// An interval represents the start (0-based, inclusive) and end (0-based exclusive) ranges of the +/// query. +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize)] +pub struct Interval { + pub start: Option, + pub end: Option, +} + +impl Interval { + #[instrument(level = "trace", skip_all, ret)] + pub fn into_one_based(self) -> io::Result { + Ok(match (self.start, self.end) { + (None, None) => NoodlesInterval::from(..), + (None, Some(end)) => NoodlesInterval::from(..=Self::convert_end(end)?), + (Some(start), None) => NoodlesInterval::from(Self::convert_start(start)?..), + (Some(start), Some(end)) => { + NoodlesInterval::from(Self::convert_start(start)?..=Self::convert_end(end)?) + } + }) + } + + /// Convert a start position to a noodles Position. + pub fn convert_start(start: u32) -> io::Result { + Self::convert_position(start, |value| { + value.checked_add(1).ok_or_else(|| { + io::Error::new( + Other, + format!("could not convert {} to 1-based position.", value), + ) + }) + }) + } + + /// Convert an end position to a noodles Position. + pub fn convert_end(end: u32) -> io::Result { + Self::convert_position(end, Ok) + } + + /// Convert a u32 position to a noodles Position. + pub fn convert_position(value: u32, convert_fn: F) -> io::Result + where + F: FnOnce(u32) -> io::Result, + { + let value = convert_fn(value).map(|value| { + usize::try_from(value).map_err(|err| { + io::Error::new( + Other, + format!("could not convert `u32` to `usize`: {}", err), + ) + }) + })??; + + Position::try_from(value).map_err(|err| { + io::Error::new( + Other, + format!("could not convert `{}` into `Position`: {}", value, err), + ) + }) + } +} + +/// Possible values for the fields parameter. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub enum Fields { + /// Include all fields + All, + /// List of fields to include + List(Vec), +} + +/// Possible values for the tags parameter. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub enum Tags { + /// Include all tags + All, + /// List of tags to include + List(Vec), +} + +/// The no tags parameter. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct NoTags(pub Option>); + +/// A query contains all the parameters that can be used when requesting +/// a search for either of `reads` or `variants`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Query { + pub id: String, + pub format: Format, + pub class: Class, + /// Reference name + pub reference_name: Option, + /// The start and end positions are 0-based. [start, end) + pub interval: Interval, + pub fields: Fields, + pub tags: Tags, + pub no_tags: NoTags, +} + +impl Query { + pub fn new(id: impl Into, format: Format) -> Self { + Self { + id: id.into(), + format, + class: Class::Body, + reference_name: None, + interval: Interval::default(), + fields: Fields::All, + tags: Tags::All, + no_tags: NoTags(None), + } + } + + pub fn with_format(mut self, format: Format) -> Self { + self.format = format; + self + } + + pub fn with_class(mut self, class: Class) -> Self { + self.class = class; + self + } + + pub fn with_reference_name(mut self, reference_name: impl Into) -> Self { + self.reference_name = Some(reference_name.into()); + self + } + + pub fn with_start(mut self, start: u32) -> Self { + self.interval.start = Some(start); + self + } + + pub fn with_end(mut self, end: u32) -> Self { + self.interval.end = Some(end); + self + } + + pub fn with_fields(mut self, fields: Fields) -> Self { + self.fields = fields; + self + } + + pub fn with_tags(mut self, tags: Tags) -> Self { + self.tags = tags; + self + } + + pub fn with_no_tags(mut self, no_tags: Vec>) -> Self { + self.no_tags = NoTags(Some( + no_tags.into_iter().map(|field| field.into()).collect(), + )); + self + } +} diff --git a/htsget-config/src/regex_resolver.rs b/htsget-config/src/regex_resolver.rs index a7d37034a..83aa2c6e0 100644 --- a/htsget-config/src/regex_resolver.rs +++ b/htsget-config/src/regex_resolver.rs @@ -1,3 +1,5 @@ +use crate::Format::{Bam, Bcf, Cram, Vcf}; +use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; use regex::{Error, Regex}; use serde::Deserialize; use tracing::instrument; @@ -5,7 +7,7 @@ use tracing::instrument; /// Represents an id resolver, which matches the id, replacing the match in the substitution text. pub trait HtsGetIdResolver { /// Resolve the id, returning the substituted string if there is a match. - fn resolve_id(&self, id: &str) -> Option; + fn resolve_id(&self, query: &Query) -> Option; } /// A regex resolver is a resolver that matches ids using Regex. @@ -13,34 +15,71 @@ pub trait HtsGetIdResolver { #[serde(default)] pub struct RegexResolver { #[serde(with = "serde_regex")] - pub(crate) regex: Regex, - pub(crate) substitution_string: String, + pub regex: Regex, + pub substitution_string: String, + #[serde(flatten)] + pub match_guard: MatchOnQuery, +} + +/// A query that can be matched with the regex resolver. +#[derive(Clone, Debug, Deserialize)] +pub struct MatchOnQuery { + pub format: Vec, + pub class: Vec, + #[serde(with = "serde_regex")] + pub reference_name: Regex, + /// The start and end positions are 0-based. [start, end) + pub start: Interval, + pub end: Interval, + pub fields: Fields, + pub tags: Tags, + pub no_tags: NoTags, +} + +impl Default for MatchOnQuery { + fn default() -> Self { + Self { + format: vec![Bam, Cram, Vcf, Bcf], + class: vec![Class::Body, Class::Header], + reference_name: Regex::new(".*").expect("Expected valid regex expression"), + start: Default::default(), + end: Default::default(), + fields: Fields::All, + tags: Tags::All, + no_tags: NoTags(None), + } + } } impl Default for RegexResolver { fn default() -> Self { - Self::new(".*", "$0").expect("expected valid resolver") + Self::new(".*", "$0", MatchOnQuery::default()).expect("expected valid resolver") } } impl RegexResolver { /// Create a new regex resolver. - pub fn new(regex: &str, replacement_string: &str) -> Result { + pub fn new( + regex: &str, + replacement_string: &str, + match_guard: MatchOnQuery, + ) -> Result { Ok(Self { regex: Regex::new(regex)?, substitution_string: replacement_string.to_string(), + match_guard, }) } } impl HtsGetIdResolver for RegexResolver { #[instrument(level = "trace", skip(self), ret)] - fn resolve_id(&self, id: &str) -> Option { - if self.regex.is_match(id) { + fn resolve_id(&self, query: &Query) -> Option { + if self.regex.is_match(&query.id) { Some( self .regex - .replace(id, &self.substitution_string) + .replace(&query.id, &self.substitution_string) .to_string(), ) } else { @@ -55,7 +94,10 @@ pub mod tests { #[test] fn resolver_resolve_id() { - let resolver = RegexResolver::new(".*", "$0-test").unwrap(); - assert_eq!(resolver.resolve_id("id").unwrap(), "id-test"); + let resolver = RegexResolver::new(".*", "$0-test", MatchOnQuery::default()).unwrap(); + assert_eq!( + resolver.resolve_id(&Query::new("id", Bam)).unwrap(), + "id-test" + ); } } diff --git a/htsget-http-core/src/lib.rs b/htsget-http-core/src/lib.rs index ba7393360..1dc3934b8 100644 --- a/htsget-http-core/src/lib.rs +++ b/htsget-http-core/src/lib.rs @@ -5,7 +5,8 @@ pub use error::{HtsGetError, Result}; pub use htsget_config::config::{ Config, DataServerConfig, ServiceInfo as ConfigServiceInfo, StorageType, TicketServerConfig, }; -use htsget_search::htsget::{Query, Response}; +use htsget_config::Query; +use htsget_search::htsget::Response; pub use http_core::{get_response_for_get_request, get_response_for_post_request}; pub use post_request::{PostRequest, Region}; use query_builder::QueryBuilder; @@ -113,11 +114,12 @@ mod tests { use std::path::PathBuf; use std::sync::Arc; - use htsget_config::regex_resolver::RegexResolver; + use htsget_config::regex_resolver::{MatchOnQuery, RegexResolver}; + use htsget_config::Format; use htsget_search::htsget::HtsGet; use htsget_search::storage::data_server::HttpTicketFormatter; use htsget_search::{ - htsget::{from_storage::HtsGetFromStorage, Format, Headers, JsonResponse, Url}, + htsget::{from_storage::HtsGetFromStorage, Headers, JsonResponse, Url}, storage::local::LocalStorage, }; use htsget_test_utils::util::expected_bgzf_eof_data_url; @@ -270,7 +272,7 @@ mod tests { Arc::new(HtsGetFromStorage::new( LocalStorage::new( get_base_path(), - RegexResolver::new(".*", "$0").unwrap(), + RegexResolver::new(".*", "$0", MatchOnQuery::default()).unwrap(), HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), "".to_string(), false), ) .unwrap(), diff --git a/htsget-http-core/src/post_request.rs b/htsget-http-core/src/post_request.rs index b3f9312e5..6282308e5 100644 --- a/htsget-http-core/src/post_request.rs +++ b/htsget-http-core/src/post_request.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use tracing::instrument; -use htsget_search::htsget::Query; +use htsget_config::Query; use crate::{QueryBuilder, Result}; @@ -61,7 +61,7 @@ impl PostRequest { #[cfg(test)] mod tests { - use htsget_search::htsget::{Class, Format}; + use htsget_config::{Class, Format}; use super::*; diff --git a/htsget-http-core/src/query_builder.rs b/htsget-http-core/src/query_builder.rs index dcdb4215e..c25ec5923 100644 --- a/htsget-http-core/src/query_builder.rs +++ b/htsget-http-core/src/query_builder.rs @@ -1,6 +1,7 @@ +use htsget_config::{Class, Fields, Format, Tags}; use tracing::instrument; -use htsget_search::htsget::{Class, Fields, Format, Query, Tags}; +use htsget_config::Query; use crate::error::{HtsGetError, Result}; @@ -183,6 +184,7 @@ impl QueryBuilder { #[cfg(test)] mod tests { use super::*; + use htsget_config::NoTags; #[test] fn query_without_id() { @@ -340,7 +342,7 @@ mod tests { "part2".to_string() ]) ); - assert_eq!(query.no_tags, Some(vec!["part3".to_string()])); + assert_eq!(query.no_tags, NoTags(Some(vec!["part3".to_string()]))); } #[test] @@ -358,6 +360,6 @@ mod tests { "part2".to_string() ]) ); - assert_eq!(query.no_tags, Some(vec!["part3".to_string()])); + assert_eq!(query.no_tags, NoTags(Some(vec!["part3".to_string()]))); } } diff --git a/htsget-http-core/src/service_info.rs b/htsget-http-core/src/service_info.rs index 91ccc3c65..e102cfbc3 100644 --- a/htsget-http-core/src/service_info.rs +++ b/htsget-http-core/src/service_info.rs @@ -1,10 +1,11 @@ use std::sync::Arc; +use htsget_config::Format; use serde::{Deserialize, Serialize}; use tracing::debug; use tracing::instrument; -use htsget_search::htsget::{Format, HtsGet}; +use htsget_search::htsget::HtsGet; use crate::ConfigServiceInfo; use crate::{Endpoint, READS_FORMATS, VARIANTS_FORMATS}; diff --git a/htsget-http-lambda/src/lib.rs b/htsget-http-lambda/src/lib.rs index a07c3c37f..d115fbe13 100644 --- a/htsget-http-lambda/src/lib.rs +++ b/htsget-http-lambda/src/lib.rs @@ -205,6 +205,7 @@ mod tests { use std::sync::Arc; use async_trait::async_trait; + use htsget_config::Class; use lambda_http::http::header::HeaderName; use lambda_http::http::Uri; use lambda_http::tower::ServiceExt; @@ -215,7 +216,7 @@ mod tests { use htsget_http_core::Endpoint; use htsget_search::htsget::from_storage::HtsGetFromStorage; - use htsget_search::htsget::{Class, HtsGet}; + use htsget_search::htsget::HtsGet; use htsget_search::storage::configure_cors; use htsget_search::storage::data_server::HttpTicketFormatter; use htsget_search::storage::local::LocalStorage; diff --git a/htsget-search/benches/search_benchmarks.rs b/htsget-search/benches/search_benchmarks.rs index 3e1b6783b..d8592aa45 100644 --- a/htsget-search/benches/search_benchmarks.rs +++ b/htsget-search/benches/search_benchmarks.rs @@ -4,11 +4,13 @@ use criterion::measurement::WallTime; use criterion::{criterion_group, criterion_main, BenchmarkGroup, Criterion}; use tokio::runtime::Runtime; +use htsget_config::regex_resolver::MatchOnQuery; +use htsget_config::Class::Header; +use htsget_config::Format::{Bam, Bcf, Cram, Vcf}; +use htsget_config::Query; use htsget_search::htsget::from_storage::HtsGetFromStorage; -use htsget_search::htsget::Class::Header; -use htsget_search::htsget::Format::{Bam, Bcf, Cram, Vcf}; use htsget_search::htsget::HtsGet; -use htsget_search::htsget::{HtsGetError, Query}; +use htsget_search::htsget::HtsGetError; use htsget_search::storage::data_server::HttpTicketFormatter; use htsget_search::RegexResolver; @@ -18,7 +20,7 @@ const NUMBER_OF_SAMPLES: usize = 150; async fn perform_query(query: Query) -> Result<(), HtsGetError> { let htsget = HtsGetFromStorage::local_from( "../data", - RegexResolver::new(".*", "$0").unwrap(), + RegexResolver::new(".*", "$0", MatchOnQuery::default()).unwrap(), HttpTicketFormatter::new( "127.0.0.1:8081".parse().expect("expected valid address"), "".to_string(), diff --git a/htsget-search/src/htsget/bam_search.rs b/htsget-search/src/htsget/bam_search.rs index 1d87649a3..7402ae86f 100644 --- a/htsget-search/src/htsget/bam_search.rs +++ b/htsget-search/src/htsget/bam_search.rs @@ -56,8 +56,7 @@ where #[instrument(level = "trace", skip(self, index))] async fn get_byte_ranges_for_unmapped( &self, - id: &str, - format: &Format, + query: &Query, index: &Index, ) -> Result> { trace!("getting byte ranges for unmapped reads"); @@ -76,7 +75,7 @@ where Ok(vec![BytesPosition::default() .with_start(start.compressed()) - .with_end(self.position_at_eof(id, format).await?) + .with_end(self.position_at_eof(query).await?) .with_class(Body)]) } } @@ -110,7 +109,7 @@ where reference_name: String, index: &Index, header: &Header, - query: Query, + query: &Query, ) -> Result> { trace!("getting byte ranges for reference name"); self @@ -148,15 +147,13 @@ where query: &Query, bai_index: &Index, ) -> Result> { - self - .get_byte_ranges_for_unmapped(&query.id, &self.get_format(), bai_index) - .await + self.get_byte_ranges_for_unmapped(query, bai_index).await } async fn get_byte_ranges_for_reference_sequence( &self, ref_seq_id: usize, - query: Query, + query: &Query, index: &Index, ) -> Result> { self diff --git a/htsget-search/src/htsget/bcf_search.rs b/htsget-search/src/htsget/bcf_search.rs index 6a1eec411..3f5578a25 100644 --- a/htsget-search/src/htsget/bcf_search.rs +++ b/htsget-search/src/htsget/bcf_search.rs @@ -77,7 +77,7 @@ where reference_name: String, index: &Index, header: &Header, - query: Query, + query: &Query, ) -> Result> { trace!("getting byte ranges for reference name"); // We are assuming the order of the contigs in the header and the references sequences diff --git a/htsget-search/src/htsget/cram_search.rs b/htsget-search/src/htsget/cram_search.rs index 62959a17f..767a0dc52 100644 --- a/htsget-search/src/htsget/cram_search.rs +++ b/htsget-search/src/htsget/cram_search.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use async_trait::async_trait; use futures::StreamExt; use futures_util::stream::FuturesOrdered; +use htsget_config::Interval; use noodles::core::Position; use noodles::cram; use noodles::cram::crai; @@ -18,7 +19,7 @@ use tracing::{instrument, trace}; use crate::htsget::search::{Search, SearchAll, SearchReads}; use crate::htsget::Class::Body; -use crate::htsget::{Format, HtsGetError, Interval, Query, Result}; +use crate::htsget::{Format, HtsGetError, Query, Result}; use crate::storage::{BytesPosition, DataBlock, Storage}; // ยง 9 End of file container . @@ -44,13 +45,9 @@ where ReaderType: AsyncRead + Unpin + Send + Sync, { #[instrument(level = "trace", skip_all, ret)] - async fn get_byte_ranges_for_all( - &self, - id: String, - format: Format, - ) -> Result> { + async fn get_byte_ranges_for_all(&self, query: &Query) -> Result> { Ok(vec![ - BytesPosition::default().with_end(self.position_at_eof(&id, &format).await?) + BytesPosition::default().with_end(self.position_at_eof(query).await?) ]) } @@ -104,9 +101,7 @@ where ) -> Result> { Self::bytes_ranges_from_index( self, - &query.id, - &self.get_format(), - &query.interval, + query, index, Arc::new(|record: &Record| record.reference_sequence_id().is_none()), ) @@ -116,14 +111,12 @@ where async fn get_byte_ranges_for_reference_sequence( &self, ref_seq_id: usize, - query: Query, + query: &Query, index: &Index, ) -> Result> { Self::bytes_ranges_from_index( self, - &query.id, - &query.format, - &query.interval, + query, index, Arc::new(move |record: &Record| record.reference_sequence_id() == Some(ref_seq_id)), ) @@ -157,7 +150,7 @@ where reference_name: String, index: &Index, header: &Header, - query: Query, + query: &Query, ) -> Result> { self .get_byte_ranges_for_reference_name_reads(&reference_name, index, header, query) @@ -184,12 +177,10 @@ where } /// Get bytes ranges using the index. - #[instrument(level = "trace", skip(self, interval, crai_index, predicate))] + #[instrument(level = "trace", skip(self, crai_index, predicate))] pub async fn bytes_ranges_from_index( &self, - id: &str, - format: &Format, - interval: &Interval, + query: &Query, crai_index: &[Record], predicate: Arc, ) -> Result> @@ -203,7 +194,7 @@ where let owned_record = record.clone(); let owned_next = next.clone(); let owned_predicate = predicate.clone(); - let range = interval.clone(); + let range = query.interval.clone(); futures.push_back(tokio::spawn(async move { if owned_predicate(&owned_record) { Self::bytes_ranges_for_record(range, &owned_record, owned_next.offset()) @@ -233,9 +224,9 @@ where } Some(last) if predicate(last) => { if let Some(range) = Self::bytes_ranges_for_record( - interval.clone(), + query.interval.clone(), last, - self.position_at_eof(id, format).await?, + self.position_at_eof(query).await?, )? { byte_ranges.push(range); } diff --git a/htsget-search/src/htsget/from_storage.rs b/htsget-search/src/htsget/from_storage.rs index 4c257a4ae..751d6e0e1 100644 --- a/htsget-search/src/htsget/from_storage.rs +++ b/htsget-search/src/htsget/from_storage.rs @@ -98,6 +98,7 @@ pub(crate) mod tests { use std::future::Future; use std::path::PathBuf; + use htsget_config::regex_resolver::MatchOnQuery; use tempfile::TempDir; use htsget_test_utils::util::expected_bgzf_eof_data_url; @@ -178,7 +179,7 @@ pub(crate) mod tests { test(Arc::new( LocalStorage::new( base_path, - RegexResolver::new(".*", "$0").unwrap(), + RegexResolver::new(".*", "$0", MatchOnQuery::default()).unwrap(), HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), "".to_string(), false), ) .unwrap(), diff --git a/htsget-search/src/htsget/mod.rs b/htsget-search/src/htsget/mod.rs index d4bcbabce..66bb06b9e 100644 --- a/htsget-search/src/htsget/mod.rs +++ b/htsget-search/src/htsget/mod.rs @@ -3,19 +3,15 @@ //! Based on the [HtsGet Specification](https://samtools.github.io/hts-specs/htsget.html). //! -use core::fmt; use std::collections::HashMap; -use std::fmt::Formatter; use std::io; use std::io::ErrorKind; use async_trait::async_trait; -use noodles::core::region::Interval as NoodlesInterval; -use noodles::core::Position; +use htsget_config::{Class, Format, Query}; use serde::{Deserialize, Serialize}; use thiserror::Error; use tokio::task::JoinError; -use tracing::instrument; use crate::storage::StorageError; @@ -127,216 +123,6 @@ impl From for HtsGetError { } } -/// A query contains all the parameters that can be used when requesting -/// a search for either of `reads` or `variants`. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Query { - pub id: String, - pub format: Format, - pub class: Class, - /// Reference name - pub reference_name: Option, - /// The start and end positions are 0-based. [start, end) - pub interval: Interval, - pub fields: Fields, - pub tags: Tags, - pub no_tags: Option>, -} - -impl Query { - pub fn new(id: impl Into, format: Format) -> Self { - Self { - id: id.into(), - format, - class: Class::Body, - reference_name: None, - interval: Interval::default(), - fields: Fields::All, - tags: Tags::All, - no_tags: None, - } - } - - pub fn with_format(mut self, format: Format) -> Self { - self.format = format; - self - } - - pub fn with_class(mut self, class: Class) -> Self { - self.class = class; - self - } - - pub fn with_reference_name(mut self, reference_name: impl Into) -> Self { - self.reference_name = Some(reference_name.into()); - self - } - - pub fn with_start(mut self, start: u32) -> Self { - self.interval.start = Some(start); - self - } - - pub fn with_end(mut self, end: u32) -> Self { - self.interval.end = Some(end); - self - } - - pub fn with_fields(mut self, fields: Fields) -> Self { - self.fields = fields; - self - } - - pub fn with_tags(mut self, tags: Tags) -> Self { - self.tags = tags; - self - } - - pub fn with_no_tags(mut self, no_tags: Vec>) -> Self { - self.no_tags = Some(no_tags.into_iter().map(|field| field.into()).collect()); - self - } -} - -/// An interval represents the start (0-based, inclusive) and end (0-based exclusive) ranges of the -/// query. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Interval { - pub start: Option, - pub end: Option, -} - -impl Interval { - #[instrument(level = "trace", skip_all, ret)] - pub fn into_one_based(self) -> Result { - Ok(match (self.start, self.end) { - (None, None) => NoodlesInterval::from(..), - (None, Some(end)) => NoodlesInterval::from(..=Self::convert_end(end)?), - (Some(start), None) => NoodlesInterval::from(Self::convert_start(start)?..), - (Some(start), Some(end)) => { - NoodlesInterval::from(Self::convert_start(start)?..=Self::convert_end(end)?) - } - }) - } - - /// Convert a start position to a noodles Position. - pub fn convert_start(start: u32) -> Result { - Self::convert_position(start, |value| { - value.checked_add(1).ok_or_else(|| { - HtsGetError::InvalidRange(format!("could not convert {} to 1-based position.", value)) - }) - }) - } - - /// Convert an end position to a noodles Position. - pub fn convert_end(end: u32) -> Result { - Self::convert_position(end, Ok) - } - - /// Convert a u32 position to a noodles Position. - pub fn convert_position(value: u32, convert_fn: F) -> Result - where - F: FnOnce(u32) -> Result, - { - let value = convert_fn(value).map(|value| { - usize::try_from(value).map_err(|err| { - HtsGetError::InvalidRange(format!("could not convert `u32` to `usize`: {}", err)) - }) - })??; - - Position::try_from(value).map_err(|err| { - HtsGetError::InvalidRange(format!( - "could not convert `{}` into `Position`: {}", - value, err - )) - }) - } -} - -/// An enumeration with all the possible formats. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "UPPERCASE")] -pub enum Format { - Bam, - Cram, - Vcf, - Bcf, -} - -// TODO Allow the user to change this. -impl Format { - pub(crate) fn fmt_file(&self, id: &str) -> String { - match self { - Format::Bam => format!("{}.bam", id), - Format::Cram => format!("{}.cram", id), - Format::Vcf => format!("{}.vcf.gz", id), - Format::Bcf => format!("{}.bcf", id), - } - } - - pub(crate) fn fmt_index(&self, id: &str) -> String { - match self { - Format::Bam => format!("{}.bam.bai", id), - Format::Cram => format!("{}.cram.crai", id), - Format::Vcf => format!("{}.vcf.gz.tbi", id), - Format::Bcf => format!("{}.bcf.csi", id), - } - } - - pub(crate) fn fmt_gzi(&self, id: &str) -> Result { - match self { - Format::Bam => Ok(format!("{}.bam.gzi", id)), - Format::Cram => Err(HtsGetError::InternalError( - "CRAM does not support GZI".to_string(), - )), - Format::Vcf => Ok(format!("{}.vcf.gz.gzi", id)), - Format::Bcf => Ok(format!("{}.bcf.gzi", id)), - } - } -} - -impl From for String { - fn from(format: Format) -> Self { - format.to_string() - } -} - -impl fmt::Display for Format { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - Format::Bam => write!(f, "BAM"), - Format::Cram => write!(f, "CRAM"), - Format::Vcf => write!(f, "VCF"), - Format::Bcf => write!(f, "BCF"), - } - } -} - -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum Class { - Header, - Body, -} - -/// Possible values for the fields parameter. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Fields { - /// Include all fields - All, - /// List of fields to include - List(Vec), -} - -/// Possible values for the tags parameter. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Tags { - /// Include all tags - All, - /// List of tags to include - List(Vec), -} - /// The headers that need to be supplied when requesting data from a url. #[derive(Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct Headers(HashMap); @@ -436,6 +222,7 @@ impl Response { #[cfg(test)] mod tests { use super::*; + use htsget_config::{Fields, NoTags, Tags}; #[test] fn htsget_error_not_found() { @@ -548,7 +335,7 @@ mod tests { let result = Query::new("NA12878", Format::Bam).with_no_tags(vec!["RG", "OQ"]); assert_eq!( result.no_tags, - Some(vec!["RG".to_string(), "OQ".to_string()]) + NoTags(Some(vec!["RG".to_string(), "OQ".to_string()])) ); } diff --git a/htsget-search/src/htsget/search.rs b/htsget-search/src/htsget/search.rs index 6b8c9be10..80df3e653 100644 --- a/htsget-search/src/htsget/search.rs +++ b/htsget-search/src/htsget/search.rs @@ -70,8 +70,7 @@ where Index: Send + Sync, { /// This returns mapped and placed unmapped ranges. - async fn get_byte_ranges_for_all(&self, id: String, format: Format) - -> Result>; + async fn get_byte_ranges_for_all(&self, query: &Query) -> Result>; /// Get the offset in the file of the end of the header. async fn get_header_end_offset(&self, index: &Index) -> Result; @@ -131,7 +130,7 @@ where async fn get_byte_ranges_for_reference_sequence( &self, ref_seq_id: usize, - query: Query, + query: &Query, index: &Index, ) -> Result>; @@ -141,10 +140,10 @@ where reference_name: &str, index: &Index, header: &Header, - query: Query, + query: &Query, ) -> Result> { if reference_name == "*" { - return self.get_byte_ranges_for_unmapped_reads(&query, index).await; + return self.get_byte_ranges_for_unmapped_reads(query, index).await; } let maybe_ref_seq = self @@ -194,7 +193,7 @@ where reference_name: String, index: &Index, header: &Header, - query: Query, + query: &Query, ) -> Result>; /// Get the storage of this format. @@ -205,8 +204,8 @@ where /// Get the position at the end of file marker. #[instrument(level = "trace", skip(self), ret)] - async fn position_at_eof(&self, id: &str, format: &Format) -> Result { - let file_size = self.get_storage().head(format.fmt_file(id)).await?; + async fn position_at_eof(&self, query: &Query) -> Result { + let file_size = self.get_storage().head(query).await?; Ok( file_size - u64::try_from(self.get_eof_marker().len()) @@ -216,12 +215,9 @@ where /// Read the index from the key. #[instrument(level = "trace", skip(self))] - async fn read_index(&self, id: &str) -> Result { + async fn read_index(&self, query: &Query) -> Result { trace!("reading index"); - let storage = self - .get_storage() - .get(self.get_format().fmt_index(id), GetOptions::default()) - .await?; + let storage = self.get_storage().get(query, GetOptions::default()).await?; Self::read_index_inner(storage) .await .map_err(|err| HtsGetError::io_error(format!("reading {} index: {}", self.get_format(), err))) @@ -239,19 +235,14 @@ where ))); } - let id = query.id.clone(); let byte_ranges = match query.reference_name.as_ref() { - None => { - self - .get_byte_ranges_for_all(query.id.clone(), format) - .await? - } + None => self.get_byte_ranges_for_all(&query).await?, Some(reference_name) => { - let index = self.read_index(&query.id).await?; - let header = self.get_header(&id, &format, &index).await?; + let index = self.read_index(&query).await?; + let header = self.get_header(&query, &index).await?; let mut byte_ranges = self - .get_byte_ranges_for_reference_name(reference_name.clone(), &index, &header, query) + .get_byte_ranges_for_reference_name(reference_name.clone(), &index, &header, &query) .await?; byte_ranges.push(self.get_byte_ranges_for_header(&index).await?); @@ -264,16 +255,15 @@ where blocks.push(eof); } - self.build_response(id, format, blocks).await + self.build_response(&query, blocks).await } Class::Header => { - let index = self.read_index(&query.id).await?; + let index = self.read_index(&query).await?; let header_byte_ranges = self.get_byte_ranges_for_header(&index).await?; self .build_response( - query.id, - self.get_format(), + &query, DataBlock::from_bytes_positions(vec![header_byte_ranges]), ) .await @@ -283,22 +273,17 @@ where /// Build the response from the query using urls. #[instrument(level = "trace", skip(self, byte_ranges))] - async fn build_response( - &self, - id: String, - format: Format, - byte_ranges: Vec, - ) -> Result { + async fn build_response(&self, query: &Query, byte_ranges: Vec) -> Result { trace!("building response"); let mut storage_futures = FuturesOrdered::new(); for block in DataBlock::update_classes(byte_ranges) { match block { DataBlock::Range(range) => { let storage = self.get_storage(); - let id = id.clone(); + let query_owned = query.clone(); storage_futures.push_back(tokio::spawn(async move { storage - .range_url(format.fmt_file(&id), RangeUrlOptions::from(range)) + .range_url(&query_owned, RangeUrlOptions::from(range)) .await })); } @@ -314,19 +299,16 @@ where else => break } } - return Ok(Response::new(format, urls)); + return Ok(Response::new(query.format, urls)); } /// Get the header from the file specified by the id and format. #[instrument(level = "trace", skip(self, index))] - async fn get_header(&self, id: &str, format: &Format, index: &Index) -> Result
{ + async fn get_header(&self, query: &Query, index: &Index) -> Result
{ trace!("getting header"); let get_options = GetOptions::default().with_range(self.get_byte_ranges_for_header(index).await?); - let reader_type = self - .get_storage() - .get(format.fmt_file(id), get_options) - .await?; + let reader_type = self.get_storage().get(query, get_options).await?; let mut reader = Self::init_reader(reader_type); Self::read_raw_header(&mut reader) @@ -398,14 +380,14 @@ where #[instrument(level = "trace", skip_all)] async fn get_byte_ranges_for_reference_sequence_bgzf( &self, - query: Query, + query: &Query, ref_seq_id: usize, index: &Index, ) -> Result> { let chunks: Result> = trace_span!("querying chunks").in_scope(|| { trace!(id = ?query.id.as_str(), ref_seq_id = ?ref_seq_id, "querying chunks"); let mut chunks = index - .query(ref_seq_id, query.interval.into_one_based()?) + .query(ref_seq_id, query.interval.clone().into_one_based()?) .map_err(|err| HtsGetError::InvalidRange(format!("querying range: {}", err)))?; if chunks.is_empty() { @@ -420,10 +402,7 @@ where Ok(chunks) }); - let gzi_data = self - .get_storage() - .get(self.get_format().fmt_gzi(&query.id)?, GetOptions::default()) - .await; + let gzi_data = self.get_storage().get(query, GetOptions::default()).await; let byte_ranges: Vec = match gzi_data { Ok(gzi_data) => { let span = trace_span!("reading gzi"); @@ -444,19 +423,13 @@ where .await; self - .bytes_positions_from_chunks( - &query.id, - &query.format, - chunks?.into_iter(), - gzi?.into_iter(), - ) + .bytes_positions_from_chunks(query, chunks?.into_iter(), gzi?.into_iter()) .await? } Err(_) => { self .bytes_positions_from_chunks( - &query.id, - &query.format, + query, chunks?.into_iter(), Self::index_positions(index).into_iter(), ) @@ -471,8 +444,7 @@ where #[instrument(level = "trace", skip(self, chunks, positions))] async fn bytes_positions_from_chunks<'a>( &self, - id: &str, - format: &Format, + query: &Query, chunks: impl Iterator + Send + 'a, mut positions: impl Iterator + Send + 'a, ) -> Result> { @@ -505,7 +477,7 @@ where let end = match maybe_end { None => match end_position { None => { - let pos = self.position_at_eof(id, format).await?; + let pos = self.position_at_eof(query).await?; end_position = Some(pos); pos } @@ -523,8 +495,7 @@ where /// Get unmapped bytes ranges. async fn get_byte_ranges_for_unmapped( &self, - _id: &str, - _format: &Format, + _query: &Query, _index: &Index, ) -> Result> { Ok(Vec::new()) @@ -545,13 +516,9 @@ where T: BgzfSearch + Send + Sync, { #[instrument(level = "debug", skip(self), ret)] - async fn get_byte_ranges_for_all( - &self, - id: String, - format: Format, - ) -> Result> { + async fn get_byte_ranges_for_all(&self, query: &Query) -> Result> { Ok(vec![ - BytesPosition::default().with_end(self.position_at_eof(&id, &format).await?) + BytesPosition::default().with_end(self.position_at_eof(query).await?) ]) } diff --git a/htsget-search/src/htsget/vcf_search.rs b/htsget-search/src/htsget/vcf_search.rs index e2bc8b547..c0bcbd1a5 100644 --- a/htsget-search/src/htsget/vcf_search.rs +++ b/htsget-search/src/htsget/vcf_search.rs @@ -78,7 +78,7 @@ where reference_name: String, index: &Index, _header: &Header, - query: Query, + query: &Query, ) -> Result> { trace!("getting byte ranges for reference name"); // We are assuming the order of the names and the references sequences diff --git a/htsget-search/src/storage/aws.rs b/htsget-search/src/storage/aws.rs index 2958c0d11..3f503746b 100644 --- a/htsget-search/src/storage/aws.rs +++ b/htsget-search/src/storage/aws.rs @@ -11,6 +11,7 @@ use aws_sdk_s3::types::ByteStream; use aws_sdk_s3::Client; use bytes::Bytes; use fluent_builders::GetObject; +use htsget_config::Query; use tokio_util::io::StreamReader; use tracing::debug; use tracing::instrument; @@ -61,45 +62,41 @@ impl AwsS3Storage { ) } - pub async fn s3_presign_url + Send>( - &self, - key: K, - range: BytesPosition, - ) -> Result { + pub async fn s3_presign_url(&self, query: &Query, range: BytesPosition) -> Result { let response = self .client .get_object() .bucket(&self.bucket) - .key(resolve_id(&self.id_resolver, &key)?); + .key(resolve_id(&self.id_resolver, query)?); let response = Self::apply_range(response, range); Ok( response .presigned( PresigningConfig::expires_in(Duration::from_secs(Self::PRESIGNED_REQUEST_EXPIRY)) - .map_err(|err| AwsS3Error(err.to_string(), key.as_ref().to_string()))?, + .map_err(|err| AwsS3Error(err.to_string(), query.id.to_string()))?, ) .await - .map_err(|err| AwsS3Error(err.to_string(), key.as_ref().to_string()))? + .map_err(|err| AwsS3Error(err.to_string(), query.id.to_string()))? .uri() .to_string(), ) } - async fn s3_head + Send>(&self, key: K) -> Result { + async fn s3_head(&self, query: &Query) -> Result { self .client .head_object() .bucket(&self.bucket) - .key(resolve_id(&self.id_resolver, &key)?) + .key(resolve_id(&self.id_resolver, query)?) .send() .await - .map_err(|err| AwsS3Error(err.to_string(), key.as_ref().to_string())) + .map_err(|err| AwsS3Error(err.to_string(), query.id.to_string())) } /// Returns the retrieval type of the object stored with the key. #[instrument(level = "trace", skip_all, ret)] - pub async fn get_retrieval_type + Send>(&self, key: &K) -> Result { - let head = self.s3_head(resolve_id(&self.id_resolver, &key)?).await?; + pub async fn get_retrieval_type(&self, query: &Query) -> Result { + let head = self.s3_head(query).await?; Ok( // Default is Standard. match head.storage_class.unwrap_or(StorageClass::Standard) { @@ -138,15 +135,11 @@ impl AwsS3Storage { } } - pub async fn get_content + Send>( - &self, - key: K, - options: GetOptions, - ) -> Result { - if let Delayed(class) = self.get_retrieval_type(&key).await? { + pub async fn get_content(&self, query: &Query, options: GetOptions) -> Result { + if let Delayed(class) = self.get_retrieval_type(query).await? { return Err(AwsS3Error( format!("cannot retrieve object immediately, class is `{:?}`", class), - key.as_ref().to_string(), + query.id.to_string(), )); } @@ -154,23 +147,23 @@ impl AwsS3Storage { .client .get_object() .bucket(&self.bucket) - .key(resolve_id(&self.id_resolver, &key)?); + .key(resolve_id(&self.id_resolver, query)?); let response = Self::apply_range(response, options.range); Ok( response .send() .await - .map_err(|err| AwsS3Error(err.to_string(), key.as_ref().to_string()))? + .map_err(|err| AwsS3Error(err.to_string(), query.id.to_string()))? .body, ) } - async fn create_stream_reader + Send>( + async fn create_stream_reader( &self, - key: K, + query: &Query, options: GetOptions, ) -> Result> { - let response = self.get_content(key, options).await?; + let response = self.get_content(query, options).await?; Ok(StreamReader::new(response)) } } @@ -181,40 +174,29 @@ impl Storage for AwsS3Storage { /// Gets the actual s3 object as a buffered reader. #[instrument(level = "trace", skip(self))] - async fn get + Send + Debug>( - &self, - key: K, - options: GetOptions, - ) -> Result { - let key = key.as_ref(); - debug!(calling_from = ?self, key, "getting file with key {:?}", key); + async fn get(&self, query: &Query, options: GetOptions) -> Result { + debug!(calling_from = ?self, query.id, "getting file with key {:?}", query.id); - self.create_stream_reader(key, options).await + self.create_stream_reader(query, options).await } /// Returns a S3-presigned htsget URL #[instrument(level = "trace", skip(self))] - async fn range_url + Send + Debug>( - &self, - key: K, - options: RangeUrlOptions, - ) -> Result { - let key = key.as_ref(); - let presigned_url = self.s3_presign_url(key, options.range.clone()).await?; + async fn range_url(&self, query: &Query, options: RangeUrlOptions) -> Result { + let presigned_url = self.s3_presign_url(query, options.range.clone()).await?; let url = options.apply(Url::new(presigned_url)); - debug!(calling_from = ?self, key, ?url, "getting url with key {:?}", key); + debug!(calling_from = ?self, query.id, ?url, "getting url with key {:?}", query.id); Ok(url) } /// Returns the size of the S3 object in bytes. #[instrument(level = "trace", skip(self))] - async fn head + Send + Debug>(&self, key: K) -> Result { - let key = key.as_ref(); - let head = self.s3_head(key).await?; - let len = head.content_length as u64; + async fn head(&self, query: &Query) -> Result { + let head = self.s3_head(query).await?; + let len = head.content_length as u64; // Todo fix this for safe casting - debug!(calling_from = ?self, key, len, "size of key {:?} is {}", key, len); + debug!(calling_from = ?self, query.id, len, "size of key {:?} is {}", query.id, len); Ok(len) } } @@ -230,6 +212,9 @@ mod tests { use aws_types::region::Region; use aws_types::{Credentials, SdkConfig}; use futures::future; + use htsget_config::regex_resolver::MatchOnQuery; + use htsget_config::Format::Bam; + use htsget_config::Query; use hyper::service::make_service_fn; use hyper::Server; use s3_server::storages::fs::FileSystem; @@ -289,7 +274,7 @@ mod tests { test(AwsS3Storage::new( client, folder_name, - RegexResolver::new(".*", "$0").unwrap(), + RegexResolver::new(".*", "$0", MatchOnQuery::default()).unwrap(), )); }) .await; @@ -298,7 +283,9 @@ mod tests { #[tokio::test] async fn existing_key() { with_aws_s3_storage(|storage| async move { - let result = storage.get("key2", GetOptions::default()).await; + let result = storage + .get(&Query::new("key2", Bam), GetOptions::default()) + .await; assert!(matches!(result, Ok(_))); }) .await; @@ -307,7 +294,9 @@ mod tests { #[tokio::test] async fn non_existing_key() { with_aws_s3_storage(|storage| async move { - let result = storage.get("non-existing-key", GetOptions::default()).await; + let result = storage + .get(&Query::new("non-existing-key", Bam), GetOptions::default()) + .await; assert!(matches!(result, Err(StorageError::AwsS3Error(_, _)))); }) .await; @@ -317,7 +306,10 @@ mod tests { async fn url_of_non_existing_key() { with_aws_s3_storage(|storage| async move { let result = storage - .range_url("non-existing-key", RangeUrlOptions::default()) + .range_url( + &Query::new("non-existing-key", Bam), + RangeUrlOptions::default(), + ) .await; assert!(matches!(result, Err(StorageError::AwsS3Error(_, _)))); }) @@ -328,7 +320,7 @@ mod tests { async fn url_of_existing_key() { with_aws_s3_storage(|storage| async move { let result = storage - .range_url("key2", RangeUrlOptions::default()) + .range_url(&Query::new("key2", Bam), RangeUrlOptions::default()) .await .unwrap(); assert!(result @@ -347,7 +339,7 @@ mod tests { with_aws_s3_storage(|storage| async move { let result = storage .range_url( - "key2", + &Query::new("key2", Bam), RangeUrlOptions::default().with_range(BytesPosition::new(Some(7), Some(9), None)), ) .await @@ -373,7 +365,7 @@ mod tests { with_aws_s3_storage(|storage| async move { let result = storage .range_url( - "key2", + &Query::new("key2", Bam), RangeUrlOptions::default().with_range(BytesPosition::new(Some(7), None, None)), ) .await @@ -397,7 +389,7 @@ mod tests { #[tokio::test] async fn file_size() { with_aws_s3_storage(|storage| async move { - let result = storage.head("key2").await; + let result = storage.head(&Query::new("key2", Bam)).await; let expected: u64 = 6; assert!(matches!(result, Ok(size) if size == expected)); }) @@ -407,7 +399,7 @@ mod tests { #[tokio::test] async fn retrieval_type() { with_aws_s3_storage(|storage| async move { - let result = storage.get_retrieval_type(&"key2".to_string()).await; + let result = storage.get_retrieval_type(&Query::new("key2", Bam)).await; println!("{:?}", result); }) .await; diff --git a/htsget-search/src/storage/local.rs b/htsget-search/src/storage/local.rs index bf19eae8b..46ab84084 100644 --- a/htsget-search/src/storage/local.rs +++ b/htsget-search/src/storage/local.rs @@ -5,6 +5,7 @@ use std::fmt::Debug; use std::path::{Path, PathBuf}; use async_trait::async_trait; +use htsget_config::Query; use tokio::fs::File; use tracing::debug; use tracing::instrument; @@ -46,32 +47,31 @@ impl LocalStorage { self.base_path.as_path() } - pub(crate) fn get_path_from_key>(&self, key: K) -> Result { - let key: &str = key.as_ref(); + pub(crate) fn get_path_from_key(&self, query: &Query) -> Result { self .base_path - .join(resolve_id(&self.id_resolver, &key)?) + .join(resolve_id(&self.id_resolver, query)?) .canonicalize() - .map_err(|_| StorageError::InvalidKey(key.to_string())) + .map_err(|_| StorageError::InvalidKey(query.id.to_string())) .and_then(|path| { path .starts_with(&self.base_path) .then_some(path) - .ok_or_else(|| StorageError::InvalidKey(key.to_string())) + .ok_or_else(|| StorageError::InvalidKey(query.id.to_string())) }) .and_then(|path| { path .is_file() .then_some(path) - .ok_or_else(|| StorageError::KeyNotFound(key.to_string())) + .ok_or_else(|| StorageError::KeyNotFound(query.id.to_string())) }) } - pub async fn get>(&self, key: K) -> Result { - let path = self.get_path_from_key(&key)?; + pub async fn get(&self, query: &Query) -> Result { + let path = self.get_path_from_key(query)?; File::open(path) .await - .map_err(|_| StorageError::KeyNotFound(key.as_ref().to_string())) + .map_err(|_| StorageError::KeyNotFound(query.id.to_string())) } } @@ -81,19 +81,15 @@ impl Storage for LocalStorage { /// Get the file at the location of the key. #[instrument(level = "debug", skip(self))] - async fn get + Send + Debug>(&self, key: K, _options: GetOptions) -> Result { - debug!(calling_from = ?self, key = key.as_ref(), "getting file with key {:?}", key.as_ref()); - self.get(key).await + async fn get(&self, query: &Query, _options: GetOptions) -> Result { + debug!(calling_from = ?self, id = query.id, "getting file with key {:?}", query.id); + self.get(query).await } /// Get a url for the file at key. #[instrument(level = "debug", skip(self))] - async fn range_url + Send + Debug>( - &self, - key: K, - options: RangeUrlOptions, - ) -> Result { - let path = self.get_path_from_key(&key)?; + async fn range_url(&self, query: &Query, options: RangeUrlOptions) -> Result { + let path = self.get_path_from_key(query)?; let path = path .strip_prefix(&self.base_path) .map_err(|err| StorageError::InternalError(err.to_string()))? @@ -102,20 +98,20 @@ impl Storage for LocalStorage { let url = Url::new(self.url_formatter.format_url(&path)?); let url = options.apply(url); - debug!(calling_from = ?self, key = key.as_ref(), ?url, "getting url with key {:?}", key.as_ref()); + debug!(calling_from = ?self, id = query.id, ?url, "getting url with key {:?}", query.id); Ok(url) } /// Get the size of the file. #[instrument(level = "debug", skip(self))] - async fn head + Send + Debug>(&self, key: K) -> Result { - let path = self.get_path_from_key(&key)?; + async fn head(&self, query: &Query) -> Result { + let path = self.get_path_from_key(query)?; let len = tokio::fs::metadata(path) .await .map_err(|err| StorageError::KeyNotFound(err.to_string()))? .len(); - debug!(calling_from = ?self, key = key.as_ref(), len, "size of key {:?} is {}", key.as_ref(), len); + debug!(calling_from = ?self, id = query.id, len, "size of key {:?} is {}", query.id, len); Ok(len) } } @@ -125,6 +121,8 @@ pub(crate) mod tests { use std::future::Future; use std::matches; + use htsget_config::regex_resolver::MatchOnQuery; + use htsget_config::Format::Bam; use tempfile::TempDir; use tokio::fs::{create_dir, File}; use tokio::io::AsyncWriteExt; @@ -138,7 +136,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_non_existing_key() { with_local_storage(|storage| async move { - let result = storage.get("non-existing-key").await; + let result = storage.get(&Query::new("non-existing-key", Bam)).await; assert!(matches!(result, Err(StorageError::InvalidKey(msg)) if msg == "non-existing-key")); }) .await; @@ -147,7 +145,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_folder() { with_local_storage(|storage| async move { - let result = Storage::get(&storage, "folder", GetOptions::default()).await; + let result = Storage::get(&storage, &Query::new("folder", Bam), GetOptions::default()).await; assert!(matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "folder")); }) .await; @@ -156,7 +154,12 @@ pub(crate) mod tests { #[tokio::test] async fn get_forbidden_path() { with_local_storage(|storage| async move { - let result = Storage::get(&storage, "folder/../../passwords", GetOptions::default()).await; + let result = Storage::get( + &storage, + &Query::new("folder/../../passwords", Bam), + GetOptions::default(), + ) + .await; assert!( matches!(result, Err(StorageError::InvalidKey(msg)) if msg == "folder/../../passwords") ); @@ -167,7 +170,12 @@ pub(crate) mod tests { #[tokio::test] async fn get_existing_key() { with_local_storage(|storage| async move { - let result = Storage::get(&storage, "folder/../key1", GetOptions::default()).await; + let result = Storage::get( + &storage, + &Query::new("folder/../key1", Bam), + GetOptions::default(), + ) + .await; assert!(matches!(result, Ok(_))); }) .await; @@ -176,8 +184,12 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_non_existing_key() { with_local_storage(|storage| async move { - let result = - Storage::range_url(&storage, "non-existing-key", RangeUrlOptions::default()).await; + let result = Storage::range_url( + &storage, + &Query::new("non-existing-key", Bam), + RangeUrlOptions::default(), + ) + .await; assert!(matches!(result, Err(StorageError::InvalidKey(msg)) if msg == "non-existing-key")); }) .await; @@ -186,7 +198,12 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_folder() { with_local_storage(|storage| async move { - let result = Storage::range_url(&storage, "folder", RangeUrlOptions::default()).await; + let result = Storage::range_url( + &storage, + &Query::new("folder", Bam), + RangeUrlOptions::default(), + ) + .await; assert!(matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "folder")); }) .await; @@ -197,7 +214,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::range_url( &storage, - "folder/../../passwords", + &Query::new("folder/../../passwords", Bam), RangeUrlOptions::default(), ) .await; @@ -211,7 +228,12 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_existing_key() { with_local_storage(|storage| async move { - let result = Storage::range_url(&storage, "folder/../key1", RangeUrlOptions::default()).await; + let result = Storage::range_url( + &storage, + &Query::new("folder/../key1", Bam), + RangeUrlOptions::default(), + ) + .await; let expected = Url::new("http://127.0.0.1:8081/data/key1"); assert!(matches!(result, Ok(url) if url == expected)); }) @@ -223,7 +245,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::range_url( &storage, - "folder/../key1", + &Query::new("folder/../key1", Bam), RangeUrlOptions::default().with_range(BytesPosition::new(Some(7), Some(10), None)), ) .await; @@ -239,7 +261,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::range_url( &storage, - "folder/../key1", + &Query::new("folder/../key1", Bam), RangeUrlOptions::default().with_range(BytesPosition::new(Some(7), None, None)), ) .await; @@ -253,7 +275,7 @@ pub(crate) mod tests { #[tokio::test] async fn file_size() { with_local_storage(|storage| async move { - let result = Storage::head(&storage, "folder/../key1").await; + let result = Storage::head(&storage, &Query::new("folder/../key1", Bam)).await; let expected: u64 = 6; assert!(matches!(result, Ok(size) if size == expected)); }) @@ -296,7 +318,7 @@ pub(crate) mod tests { test( LocalStorage::new( base_path.path(), - RegexResolver::new(".*", "$0").unwrap(), + RegexResolver::new(".*", "$0", MatchOnQuery::default()).unwrap(), HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), "".to_string(), false), ) .unwrap(), diff --git a/htsget-search/src/storage/mod.rs b/htsget-search/src/storage/mod.rs index d922da091..b7f74f806 100644 --- a/htsget-search/src/storage/mod.rs +++ b/htsget-search/src/storage/mod.rs @@ -9,13 +9,14 @@ use std::time::Duration; use async_trait::async_trait; use base64::encode; +use htsget_config::{Class, Query}; use http::{HeaderValue, Method}; use thiserror::Error; use tokio::io::AsyncRead; use tower_http::cors::{AllowHeaders, AllowMethods, CorsLayer}; use tracing::instrument; -use crate::htsget::{Class, Headers, Url}; +use crate::htsget::{Headers, Url}; use crate::storage::data_server::CORS_MAX_AGE; use crate::storage::StorageError::DataServerError; use crate::{HtsGetIdResolver, RegexResolver}; @@ -34,21 +35,13 @@ pub trait Storage { type Streamable: AsyncRead + Unpin + Send; /// Get the object using the key. - async fn get + Send + Debug>( - &self, - key: K, - options: GetOptions, - ) -> Result; + async fn get(&self, query: &Query, options: GetOptions) -> Result; /// Get the url of the object represented by the key using a bytes range. - async fn range_url + Send + Debug>( - &self, - key: K, - options: RangeUrlOptions, - ) -> Result; + async fn range_url(&self, query: &Query, options: RangeUrlOptions) -> Result; /// Get the size of the object represented by the key. - async fn head + Send + Debug>(&self, key: K) -> Result; + async fn head(&self, query: &Query) -> Result; /// Get the url of the object using an inline data uri. #[instrument(level = "trace", ret)] @@ -375,17 +368,16 @@ impl RangeUrlOptions { } /// Resolve a key id with the `RegexResolver` and convert it to a Result. -fn resolve_id>(resolver: &RegexResolver, key: &K) -> Result { +fn resolve_id(resolver: &RegexResolver, query: &Query) -> Result { resolver - .resolve_id(key.as_ref()) - .ok_or_else(|| StorageError::InvalidKey(key.as_ref().to_string())) + .resolve_id(query) + .ok_or_else(|| StorageError::InvalidKey(query.id.to_string())) } #[cfg(test)] mod tests { use std::collections::HashMap; - use crate::htsget::Class; use crate::storage::data_server::HttpTicketFormatter; use crate::storage::local::LocalStorage; diff --git a/htsget-test-utils/src/server_tests.rs b/htsget-test-utils/src/server_tests.rs index b5a8c1404..16a84fd49 100644 --- a/htsget-test-utils/src/server_tests.rs +++ b/htsget-test-utils/src/server_tests.rs @@ -3,15 +3,15 @@ use std::path::PathBuf; use futures::future::join_all; use futures::TryStreamExt; +use htsget_config::{Class, Format}; use http::Method; use noodles_bgzf as bgzf; use noodles_vcf as vcf; use reqwest::ClientBuilder; use htsget_http_core::{get_service_info_with, Endpoint}; -use htsget_search::htsget::Class::Body; use htsget_search::htsget::Response as HtsgetResponse; -use htsget_search::htsget::{Class, Format, Headers, JsonResponse, Url}; +use htsget_search::htsget::{Headers, JsonResponse, Url}; use htsget_search::storage::data_server::HttpTicketFormatter; use crate::http_tests::{Header, Response, TestRequest, TestServer}; @@ -110,7 +110,7 @@ pub async fn test_get(tester: &impl TestServer) { .uri("/variants/vcf/sample1-bcbio-cancer"); let response = tester.test_server(request).await; - test_response(response, Body).await; + test_response(response, Class::Body).await; } fn post_request(tester: &impl TestServer) -> T { @@ -129,7 +129,7 @@ pub async fn test_post(tester: &impl TestServer) { let request = post_request(tester).set_payload("{}"); let response = tester.test_server(request).await; - test_response(response, Body).await; + test_response(response, Class::Body).await; } /// A parameterized get test. @@ -149,7 +149,7 @@ pub async fn test_parameterized_post(tester: &impl TestServer .set_payload("{\"format\": \"VCF\", \"regions\": [{\"referenceName\": \"chrM\"}]}"); let response = tester.test_server(request).await; - test_response(response, Body).await; + test_response(response, Class::Body).await; } /// A parameterized post test with header as the class. @@ -186,7 +186,7 @@ pub fn expected_response(class: Class, url_path: String) -> JsonResponse { .with_headers(Headers::new(headers)); let urls = match class { Class::Header => vec![http_url.with_class(Class::Header)], - Body => vec![http_url, Url::new(expected_bgzf_eof_data_url())], + Class::Body => vec![http_url, Url::new(expected_bgzf_eof_data_url())], }; JsonResponse::from(HtsgetResponse::new(Format::Vcf, urls)) From 8c2b3fb68f5ae9f96eac8b69f25c35f5d787f939 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 18 Nov 2022 15:34:50 +1100 Subject: [PATCH 05/45] search: add safe cast for conversion between i64 and u64 --- htsget-search/src/storage/aws.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/htsget-search/src/storage/aws.rs b/htsget-search/src/storage/aws.rs index 3f503746b..1eb3617e3 100644 --- a/htsget-search/src/storage/aws.rs +++ b/htsget-search/src/storage/aws.rs @@ -1,5 +1,7 @@ //! Module providing an implementation for the [Storage] trait using Amazon's S3 object storage service. use std::fmt::Debug; +use std::io; +use std::io::ErrorKind::Other; use std::time::Duration; use async_trait::async_trait; @@ -19,7 +21,7 @@ use tracing::instrument; use crate::htsget::Url; use crate::storage::aws::Retrieval::{Delayed, Immediate}; use crate::storage::StorageError::AwsS3Error; -use crate::storage::{resolve_id, BytesPosition}; +use crate::storage::{resolve_id, BytesPosition, StorageError}; use crate::storage::{BytesRange, Storage}; use crate::RegexResolver; @@ -194,7 +196,12 @@ impl Storage for AwsS3Storage { #[instrument(level = "trace", skip(self))] async fn head(&self, query: &Query) -> Result { let head = self.s3_head(query).await?; - let len = head.content_length as u64; // Todo fix this for safe casting + let len = u64::try_from(head.content_length).map_err(|err| { + StorageError::IoError( + "failed to convert file length to `u64`".to_string(), + io::Error::new(Other, err), + ) + })?; debug!(calling_from = ?self, query.id, len, "size of key {:?} is {}", query.id, len); Ok(len) From a4b03f1142496d0948e15b4115c6d221b3e121d1 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 22 Nov 2022 09:17:38 +1100 Subject: [PATCH 06/45] config: implement query matcher logic --- htsget-config/src/lib.rs | 16 +++++++++ htsget-config/src/regex_resolver.rs | 53 +++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 4acd5441e..23ccabcf2 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -86,6 +86,22 @@ pub struct Interval { } impl Interval { + /// Check if this interval contains the value. + pub fn contains(&self, value: Option) -> bool { + let cond1 = match (self.start.as_ref(), value.as_ref()) { + (None, _) => true, + (Some(_), None) => false, + (Some(start), Some(value)) => value >= start, + }; + let cond2 = match (self.end.as_ref(), value.as_ref()) { + (None, _) => true, + (Some(_), None) => false, + (Some(end), Some(value)) => end > value, + }; + cond1 && cond2 + } + + /// Convert this interval into a one-based noodles `Interval`. #[instrument(level = "trace", skip_all, ret)] pub fn into_one_based(self) -> io::Result { Ok(match (self.start, self.end) { diff --git a/htsget-config/src/regex_resolver.rs b/htsget-config/src/regex_resolver.rs index 83aa2c6e0..dde1e2117 100644 --- a/htsget-config/src/regex_resolver.rs +++ b/htsget-config/src/regex_resolver.rs @@ -10,6 +10,12 @@ pub trait HtsGetIdResolver { fn resolve_id(&self, query: &Query) -> Option; } +/// Determines whether the query matches for use with the resolver. +pub trait QueryMatcher { + /// Does this query match. + fn query_matches(&self, query: &Query) -> bool; +} + /// A regex resolver is a resolver that matches ids using Regex. #[derive(Debug, Clone, Deserialize)] #[serde(default)] @@ -51,6 +57,53 @@ impl Default for MatchOnQuery { } } +impl QueryMatcher for Fields { + fn query_matches(&self, query: &Query) -> bool { + match (self, &query.fields) { + (Fields::All, Fields::All) => true, + (Fields::List(self_fields), Fields::List(query_fields)) => self_fields == query_fields, + _ => false, + } + } +} + +impl QueryMatcher for Tags { + fn query_matches(&self, query: &Query) -> bool { + match (self, &query.tags) { + (Tags::All, Tags::All) => true, + (Tags::List(self_tags), Tags::List(query_tags)) => self_tags == query_tags, + _ => false, + } + } +} + +impl QueryMatcher for NoTags { + fn query_matches(&self, query: &Query) -> bool { + match (self, &query.no_tags) { + (NoTags(None), NoTags(None)) => true, + (NoTags(Some(self_no_tags)), NoTags(Some(query_no_tags))) => self_no_tags == query_no_tags, + _ => false, + } + } +} + +impl QueryMatcher for MatchOnQuery { + fn query_matches(&self, query: &Query) -> bool { + if let Some(reference_name) = &query.reference_name { + self.format.contains(&query.format) + && self.class.contains(&query.class) + && self.reference_name.is_match(reference_name) + && self.start.contains(query.interval.start) + && self.end.contains(query.interval.end) + && self.fields.query_matches(query) + && self.fields.query_matches(query) + && self.fields.query_matches(query) + } else { + false + } + } +} + impl Default for RegexResolver { fn default() -> Self { Self::new(".*", "$0", MatchOnQuery::default()).expect("expected valid resolver") From 4a1a7774be9569fc0476672009537ff7399e091b Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 22 Nov 2022 13:27:36 +1100 Subject: [PATCH 07/45] test: add tests for checking for contained value in interval --- htsget-config/src/config.rs | 18 ++++- htsget-config/src/lib.rs | 85 +++++++++++++++++++--- htsget-config/src/regex_resolver.rs | 45 ++++++++---- htsget-http-core/src/lib.rs | 9 ++- htsget-search/benches/search_benchmarks.rs | 9 ++- htsget-search/src/htsget/from_storage.rs | 11 ++- htsget-search/src/storage/aws.rs | 19 +++-- htsget-search/src/storage/local.rs | 17 ++++- 8 files changed, 173 insertions(+), 40 deletions(-) diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index a0acddbc4..68f126bbc 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -148,12 +148,26 @@ impl Default for LocalDataServer { } } +/// Specify the storage type to use. +#[derive(Deserialize, Debug, Clone)] +#[non_exhaustive] +pub enum StorageTypeServer { + LocalStorage(LocalDataServer), + #[cfg(feature = "s3-storage")] + AwsS3Storage(AwsS3DataServer), +} + +impl Default for StorageTypeServer { + fn default() -> Self { + Self::LocalStorage(LocalDataServer::default()) + } +} + /// Configuration for the htsget server. #[derive(Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct AwsS3DataServer { pub bucket: String, - pub regex_resolvers: Vec, } /// Configuration for the htsget server. @@ -165,7 +179,6 @@ pub struct DataServerConfig { pub data_server_cert: Option, pub data_server_cors_allow_credentials: bool, pub data_server_cors_allow_origin: String, - pub regex_resolvers: Vec, } /// Configuration of the service info. @@ -203,7 +216,6 @@ impl Default for DataServerConfig { data_server_cert: None, data_server_cors_allow_credentials: false, data_server_cors_allow_origin: default_data_server_origin(), - regex_resolvers: vec![RegexResolver::default()], } } } diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 23ccabcf2..b7cfb9f06 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -87,18 +87,13 @@ pub struct Interval { impl Interval { /// Check if this interval contains the value. - pub fn contains(&self, value: Option) -> bool { - let cond1 = match (self.start.as_ref(), value.as_ref()) { - (None, _) => true, - (Some(_), None) => false, - (Some(start), Some(value)) => value >= start, + pub fn contains(&self, value: u32) -> bool { + return match (self.start.as_ref(), self.end.as_ref()) { + (None, None) => true, + (None, Some(end)) => value < *end, + (Some(start), None) => value >= *start, + (Some(start), Some(end)) => value >= *start && value < *end, }; - let cond2 = match (self.end.as_ref(), value.as_ref()) { - (None, _) => true, - (Some(_), None) => false, - (Some(end), Some(value)) => end > value, - }; - cond1 && cond2 } /// Convert this interval into a one-based noodles `Interval`. @@ -248,3 +243,71 @@ impl Query { self } } + +#[cfg(test)] +mod tests { + use crate::Interval; + + #[test] + fn interval_contains() { + let interval = Interval { + start: Some(0), + end: Some(10), + }; + assert!(interval.contains(9)); + } + + #[test] + fn interval_not_contains() { + let interval = Interval { + start: Some(0), + end: Some(10), + }; + assert!(!interval.contains(10)); + } + + #[test] + fn interval_contains_start_not_present() { + let interval = Interval { + start: None, + end: Some(10), + }; + assert!(interval.contains(9)); + } + + #[test] + fn interval_not_contains_start_not_present() { + let interval = Interval { + start: None, + end: Some(10), + }; + assert!(!interval.contains(10)); + } + + #[test] + fn interval_contains_end_not_present() { + let interval = Interval { + start: Some(1), + end: None, + }; + assert!(interval.contains(9)); + } + + #[test] + fn interval_not_contains_end_not_present() { + let interval = Interval { + start: Some(1), + end: None, + }; + assert!(!interval.contains(0)); + } + + #[test] + fn interval_contains_both_not_present() { + let interval = Interval { + start: None, + end: None, + }; + assert!(interval.contains(0)); + } +} diff --git a/htsget-config/src/regex_resolver.rs b/htsget-config/src/regex_resolver.rs index dde1e2117..aa074e70a 100644 --- a/htsget-config/src/regex_resolver.rs +++ b/htsget-config/src/regex_resolver.rs @@ -1,9 +1,11 @@ -use crate::Format::{Bam, Bcf, Cram, Vcf}; -use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; use regex::{Error, Regex}; use serde::Deserialize; use tracing::instrument; +use crate::config::StorageTypeServer; +use crate::Format::{Bam, Bcf, Cram, Vcf}; +use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; + /// Represents an id resolver, which matches the id, replacing the match in the substitution text. pub trait HtsGetIdResolver { /// Resolve the id, returning the substituted string if there is a match. @@ -23,6 +25,7 @@ pub struct RegexResolver { #[serde(with = "serde_regex")] pub regex: Regex, pub substitution_string: String, + pub server: StorageTypeServer, #[serde(flatten)] pub match_guard: MatchOnQuery, } @@ -60,9 +63,9 @@ impl Default for MatchOnQuery { impl QueryMatcher for Fields { fn query_matches(&self, query: &Query) -> bool { match (self, &query.fields) { - (Fields::All, Fields::All) => true, + (Fields::All, _) => true, (Fields::List(self_fields), Fields::List(query_fields)) => self_fields == query_fields, - _ => false, + (Fields::List(_), Fields::All) => false, } } } @@ -70,9 +73,9 @@ impl QueryMatcher for Fields { impl QueryMatcher for Tags { fn query_matches(&self, query: &Query) -> bool { match (self, &query.tags) { - (Tags::All, Tags::All) => true, + (Tags::All, _) => true, (Tags::List(self_tags), Tags::List(query_tags)) => self_tags == query_tags, - _ => false, + (Tags::List(_), Tags::All) => false, } } } @@ -80,9 +83,9 @@ impl QueryMatcher for Tags { impl QueryMatcher for NoTags { fn query_matches(&self, query: &Query) -> bool { match (self, &query.no_tags) { - (NoTags(None), NoTags(None)) => true, + (NoTags(None), _) => true, (NoTags(Some(self_no_tags)), NoTags(Some(query_no_tags))) => self_no_tags == query_no_tags, - _ => false, + (NoTags(Some(_)), NoTags(None)) => false, } } } @@ -93,8 +96,10 @@ impl QueryMatcher for MatchOnQuery { self.format.contains(&query.format) && self.class.contains(&query.class) && self.reference_name.is_match(reference_name) - && self.start.contains(query.interval.start) - && self.end.contains(query.interval.end) + && self + .start + .contains(query.interval.start.unwrap_or(u32::MIN)) + && self.end.contains(query.interval.end.unwrap_or(u32::MAX)) && self.fields.query_matches(query) && self.fields.query_matches(query) && self.fields.query_matches(query) @@ -106,7 +111,13 @@ impl QueryMatcher for MatchOnQuery { impl Default for RegexResolver { fn default() -> Self { - Self::new(".*", "$0", MatchOnQuery::default()).expect("expected valid resolver") + Self::new( + ".*", + "$0", + StorageTypeServer::default(), + MatchOnQuery::default(), + ) + .expect("expected valid resolver") } } @@ -115,10 +126,12 @@ impl RegexResolver { pub fn new( regex: &str, replacement_string: &str, + server: StorageTypeServer, match_guard: MatchOnQuery, ) -> Result { Ok(Self { regex: Regex::new(regex)?, + server, substitution_string: replacement_string.to_string(), match_guard, }) @@ -128,7 +141,7 @@ impl RegexResolver { impl HtsGetIdResolver for RegexResolver { #[instrument(level = "trace", skip(self), ret)] fn resolve_id(&self, query: &Query) -> Option { - if self.regex.is_match(&query.id) { + if self.regex.is_match(&query.id) && self.match_guard.query_matches(query) { Some( self .regex @@ -147,7 +160,13 @@ pub mod tests { #[test] fn resolver_resolve_id() { - let resolver = RegexResolver::new(".*", "$0-test", MatchOnQuery::default()).unwrap(); + let resolver = RegexResolver::new( + ".*", + "$0-test", + StorageTypeServer::default(), + MatchOnQuery::default(), + ) + .unwrap(); assert_eq!( resolver.resolve_id(&Query::new("id", Bam)).unwrap(), "id-test" diff --git a/htsget-http-core/src/lib.rs b/htsget-http-core/src/lib.rs index 1dc3934b8..b52caa73f 100644 --- a/htsget-http-core/src/lib.rs +++ b/htsget-http-core/src/lib.rs @@ -114,6 +114,7 @@ mod tests { use std::path::PathBuf; use std::sync::Arc; + use htsget_config::config::StorageTypeServer; use htsget_config::regex_resolver::{MatchOnQuery, RegexResolver}; use htsget_config::Format; use htsget_search::htsget::HtsGet; @@ -272,7 +273,13 @@ mod tests { Arc::new(HtsGetFromStorage::new( LocalStorage::new( get_base_path(), - RegexResolver::new(".*", "$0", MatchOnQuery::default()).unwrap(), + RegexResolver::new( + ".*", + "$0", + StorageTypeServer::default(), + MatchOnQuery::default(), + ) + .unwrap(), HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), "".to_string(), false), ) .unwrap(), diff --git a/htsget-search/benches/search_benchmarks.rs b/htsget-search/benches/search_benchmarks.rs index d8592aa45..91d195433 100644 --- a/htsget-search/benches/search_benchmarks.rs +++ b/htsget-search/benches/search_benchmarks.rs @@ -4,6 +4,7 @@ use criterion::measurement::WallTime; use criterion::{criterion_group, criterion_main, BenchmarkGroup, Criterion}; use tokio::runtime::Runtime; +use htsget_config::config::StorageTypeServer; use htsget_config::regex_resolver::MatchOnQuery; use htsget_config::Class::Header; use htsget_config::Format::{Bam, Bcf, Cram, Vcf}; @@ -20,7 +21,13 @@ const NUMBER_OF_SAMPLES: usize = 150; async fn perform_query(query: Query) -> Result<(), HtsGetError> { let htsget = HtsGetFromStorage::local_from( "../data", - RegexResolver::new(".*", "$0", MatchOnQuery::default()).unwrap(), + RegexResolver::new( + ".*", + "$0", + StorageTypeServer::default(), + MatchOnQuery::default(), + ) + .unwrap(), HttpTicketFormatter::new( "127.0.0.1:8081".parse().expect("expected valid address"), "".to_string(), diff --git a/htsget-search/src/htsget/from_storage.rs b/htsget-search/src/htsget/from_storage.rs index 751d6e0e1..f597153a3 100644 --- a/htsget-search/src/htsget/from_storage.rs +++ b/htsget-search/src/htsget/from_storage.rs @@ -98,9 +98,10 @@ pub(crate) mod tests { use std::future::Future; use std::path::PathBuf; - use htsget_config::regex_resolver::MatchOnQuery; use tempfile::TempDir; + use htsget_config::config::StorageTypeServer; + use htsget_config::regex_resolver::MatchOnQuery; use htsget_test_utils::util::expected_bgzf_eof_data_url; use crate::htsget::bam_search::tests::{ @@ -179,7 +180,13 @@ pub(crate) mod tests { test(Arc::new( LocalStorage::new( base_path, - RegexResolver::new(".*", "$0", MatchOnQuery::default()).unwrap(), + RegexResolver::new( + ".*", + "$0", + StorageTypeServer::default(), + MatchOnQuery::default(), + ) + .unwrap(), HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), "".to_string(), false), ) .unwrap(), diff --git a/htsget-search/src/storage/aws.rs b/htsget-search/src/storage/aws.rs index 1eb3617e3..b196b1305 100644 --- a/htsget-search/src/storage/aws.rs +++ b/htsget-search/src/storage/aws.rs @@ -13,11 +13,12 @@ use aws_sdk_s3::types::ByteStream; use aws_sdk_s3::Client; use bytes::Bytes; use fluent_builders::GetObject; -use htsget_config::Query; use tokio_util::io::StreamReader; use tracing::debug; use tracing::instrument; +use htsget_config::Query; + use crate::htsget::Url; use crate::storage::aws::Retrieval::{Delayed, Immediate}; use crate::storage::StorageError::AwsS3Error; @@ -219,14 +220,16 @@ mod tests { use aws_types::region::Region; use aws_types::{Credentials, SdkConfig}; use futures::future; - use htsget_config::regex_resolver::MatchOnQuery; - use htsget_config::Format::Bam; - use htsget_config::Query; use hyper::service::make_service_fn; use hyper::Server; use s3_server::storages::fs::FileSystem; use s3_server::{S3Service, SimpleAuth}; + use htsget_config::config::StorageTypeServer; + use htsget_config::regex_resolver::MatchOnQuery; + use htsget_config::Format::Bam; + use htsget_config::Query; + use crate::htsget::Headers; use crate::storage::aws::AwsS3Storage; use crate::storage::local::tests::create_local_test_files; @@ -281,7 +284,13 @@ mod tests { test(AwsS3Storage::new( client, folder_name, - RegexResolver::new(".*", "$0", MatchOnQuery::default()).unwrap(), + RegexResolver::new( + ".*", + "$0", + StorageTypeServer::default(), + MatchOnQuery::default(), + ) + .unwrap(), )); }) .await; diff --git a/htsget-search/src/storage/local.rs b/htsget-search/src/storage/local.rs index 46ab84084..516d54f9e 100644 --- a/htsget-search/src/storage/local.rs +++ b/htsget-search/src/storage/local.rs @@ -5,11 +5,12 @@ use std::fmt::Debug; use std::path::{Path, PathBuf}; use async_trait::async_trait; -use htsget_config::Query; use tokio::fs::File; use tracing::debug; use tracing::instrument; +use htsget_config::Query; + use crate::htsget::Url; use crate::storage::{resolve_id, Storage, UrlFormatter}; use crate::RegexResolver; @@ -121,12 +122,14 @@ pub(crate) mod tests { use std::future::Future; use std::matches; - use htsget_config::regex_resolver::MatchOnQuery; - use htsget_config::Format::Bam; use tempfile::TempDir; use tokio::fs::{create_dir, File}; use tokio::io::AsyncWriteExt; + use htsget_config::config::StorageTypeServer; + use htsget_config::regex_resolver::MatchOnQuery; + use htsget_config::Format::Bam; + use crate::htsget::{Headers, Url}; use crate::storage::data_server::HttpTicketFormatter; use crate::storage::{BytesPosition, GetOptions, RangeUrlOptions, StorageError}; @@ -318,7 +321,13 @@ pub(crate) mod tests { test( LocalStorage::new( base_path.path(), - RegexResolver::new(".*", "$0", MatchOnQuery::default()).unwrap(), + RegexResolver::new( + ".*", + "$0", + StorageTypeServer::default(), + MatchOnQuery::default(), + ) + .unwrap(), HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), "".to_string(), false), ) .unwrap(), From 8f1043c0d2bb912f915889ef0076db713abc1fbc Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 23 Nov 2022 12:49:45 +1100 Subject: [PATCH 08/45] refactor: move config into separate module --- htsget-config/src/config/aws.rs | 9 ++ .../src/{config.rs => config/mod.rs} | 131 +++++++----------- htsget-config/src/regex_resolver.rs | 10 +- 3 files changed, 67 insertions(+), 83 deletions(-) create mode 100644 htsget-config/src/config/aws.rs rename htsget-config/src/{config.rs => config/mod.rs} (80%) diff --git a/htsget-config/src/config/aws.rs b/htsget-config/src/config/aws.rs new file mode 100644 index 000000000..c308607d0 --- /dev/null +++ b/htsget-config/src/config/aws.rs @@ -0,0 +1,9 @@ +use serde; +use serde::Deserialize; + +/// Configuration for the htsget server. +#[derive(Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct AwsS3DataServer { + pub bucket: String, +} \ No newline at end of file diff --git a/htsget-config/src/config.rs b/htsget-config/src/config/mod.rs similarity index 80% rename from htsget-config/src/config.rs rename to htsget-config/src/config/mod.rs index 68f126bbc..8e8e9950d 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config/mod.rs @@ -10,10 +10,14 @@ use tracing::info; use tracing::instrument; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::{fmt, EnvFilter, Registry}; +use crate::config::aws::AwsS3DataServer; use crate::config::StorageType::LocalStorage; use crate::regex_resolver::RegexResolver; +#[cfg(feature = "s3-storage")] +pub mod aws; + /// Represents a usage string for htsget-rs. pub const USAGE: &str = r#" Available environment variables: @@ -85,29 +89,12 @@ pub struct Args { config: PathBuf, } -/// Specify the storage type to use. -#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub enum StorageType { - LocalStorage, - #[cfg(feature = "s3-storage")] - AwsS3Storage, -} - /// Configuration for the server. Each field will be read from environment variables. #[derive(Deserialize, Debug, Clone)] #[serde(default)] pub struct Config { - #[serde(flatten)] - pub resolver: RegexResolver, - pub path: PathBuf, - #[serde(flatten)] pub ticket_server_config: TicketServerConfig, - pub storage_type: StorageType, - #[serde(flatten)] - pub data_server_config: DataServerConfig, - #[cfg(feature = "s3-storage")] - pub s3_bucket: String, + pub resolvers: Vec, } /// Configuration for the htsget server. @@ -151,25 +138,18 @@ impl Default for LocalDataServer { /// Specify the storage type to use. #[derive(Deserialize, Debug, Clone)] #[non_exhaustive] -pub enum StorageTypeServer { +pub enum StorageType { LocalStorage(LocalDataServer), #[cfg(feature = "s3-storage")] AwsS3Storage(AwsS3DataServer), } -impl Default for StorageTypeServer { +impl Default for StorageType { fn default() -> Self { - Self::LocalStorage(LocalDataServer::default()) + LocalStorage(LocalDataServer::default()) } } -/// Configuration for the htsget server. -#[derive(Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct AwsS3DataServer { - pub bucket: String, -} - /// Configuration for the htsget server. #[derive(Deserialize, Debug, Clone)] #[serde(default)] @@ -223,13 +203,8 @@ impl Default for DataServerConfig { impl Default for Config { fn default() -> Self { Self { - resolver: RegexResolver::default(), - path: default_path(), + resolvers: vec![RegexResolver::default()], ticket_server_config: Default::default(), - storage_type: LocalStorage, - data_server_config: Default::default(), - #[cfg(feature = "s3-storage")] - s3_bucket: "".to_string(), } } } @@ -307,42 +282,42 @@ mod tests { ); } - #[test] - fn config_data_server_cors_allow_origin() { - std::env::set_var( - "HTSGET_DATA_SERVER_CORS_ALLOW_ORIGIN", - "http://localhost:8080", - ); - let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - assert_eq!( - config.data_server_config.data_server_cors_allow_origin, - "http://localhost:8080" - ); - } - - #[test] - fn config_ticket_server_addr() { - std::env::set_var("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082"); - let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - assert_eq!( - config.data_server_config.data_server_addr, - "127.0.0.1:8082".parse().unwrap() - ); - } - - #[test] - fn config_regex() { - std::env::set_var("HTSGET_REGEX", ".+"); - let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - assert_eq!(config.resolver.regex.to_string(), ".+"); - } - - #[test] - fn config_substitution_string() { - std::env::set_var("HTSGET_SUBSTITUTION_STRING", "$0-test"); - let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - assert_eq!(config.resolver.substitution_string, "$0-test"); - } + // #[test] + // fn config_data_server_cors_allow_origin() { + // std::env::set_var( + // "HTSGET_DATA_SERVER_CORS_ALLOW_ORIGIN", + // "http://localhost:8080", + // ); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!( + // config.data_server_config.data_server_cors_allow_origin, + // "http://localhost:8080" + // ); + // } + // + // #[test] + // fn config_ticket_server_addr() { + // std::env::set_var("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082"); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!( + // config.data_server_config.data_server_addr, + // "127.0.0.1:8082".parse().unwrap() + // ); + // } + // + // #[test] + // fn config_regex() { + // std::env::set_var("HTSGET_REGEX", ".+"); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!(config.resolver.regex.to_string(), ".+"); + // } + // + // #[test] + // fn config_substitution_string() { + // std::env::set_var("HTSGET_SUBSTITUTION_STRING", "$0-test"); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!(config.resolver.substitution_string, "$0-test"); + // } #[test] fn config_service_info_id() { @@ -351,11 +326,11 @@ mod tests { assert_eq!(config.ticket_server_config.service_info.id.unwrap(), "id"); } - #[cfg(feature = "s3-storage")] - #[test] - fn config_storage_type() { - std::env::set_var("HTSGET_STORAGE_TYPE", "AwsS3Storage"); - let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - assert_eq!(config.storage_type, StorageType::AwsS3Storage); - } -} + // #[cfg(feature = "s3-storage")] + // #[test] + // fn config_storage_type() { + // std::env::set_var("HTSGET_STORAGE_TYPE", "AwsS3Storage"); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!(config.storage_type, StorageType::AwsS3Storage); + // } +} \ No newline at end of file diff --git a/htsget-config/src/regex_resolver.rs b/htsget-config/src/regex_resolver.rs index aa074e70a..c6a0c0af7 100644 --- a/htsget-config/src/regex_resolver.rs +++ b/htsget-config/src/regex_resolver.rs @@ -2,9 +2,9 @@ use regex::{Error, Regex}; use serde::Deserialize; use tracing::instrument; -use crate::config::StorageTypeServer; use crate::Format::{Bam, Bcf, Cram, Vcf}; use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; +use crate::config::StorageType; /// Represents an id resolver, which matches the id, replacing the match in the substitution text. pub trait HtsGetIdResolver { @@ -25,7 +25,7 @@ pub struct RegexResolver { #[serde(with = "serde_regex")] pub regex: Regex, pub substitution_string: String, - pub server: StorageTypeServer, + pub server: StorageType, #[serde(flatten)] pub match_guard: MatchOnQuery, } @@ -114,7 +114,7 @@ impl Default for RegexResolver { Self::new( ".*", "$0", - StorageTypeServer::default(), + StorageType::default(), MatchOnQuery::default(), ) .expect("expected valid resolver") @@ -126,7 +126,7 @@ impl RegexResolver { pub fn new( regex: &str, replacement_string: &str, - server: StorageTypeServer, + server: StorageType, match_guard: MatchOnQuery, ) -> Result { Ok(Self { @@ -163,7 +163,7 @@ pub mod tests { let resolver = RegexResolver::new( ".*", "$0-test", - StorageTypeServer::default(), + StorageType::default(), MatchOnQuery::default(), ) .unwrap(); From 505f19e10cd02a9cefee62e3085f9d1aa35a899b Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 23 Nov 2022 15:05:41 +1100 Subject: [PATCH 09/45] config: use figment instead of config because it is simpler to set defaults --- htsget-config/Cargo.toml | 2 +- htsget-config/src/config/aws.rs | 6 ++-- htsget-config/src/config/mod.rs | 44 +++++++++++++---------------- htsget-config/src/lib.rs | 8 +++--- htsget-config/src/regex_resolver.rs | 17 ++++------- 5 files changed, 33 insertions(+), 44 deletions(-) diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 214a75f32..f136b2d60 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -13,7 +13,7 @@ noodles = { version = "0.29", features = ["core"] } serde = { version = "1.0", features = ["derive"] } serde_regex = "1.1" regex = "1.6" -config = "0.13" +figment = { version = "0.10", features = ["env", "toml"] } clap = { version = "4.0", features = ["derive", "env"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] } \ No newline at end of file diff --git a/htsget-config/src/config/aws.rs b/htsget-config/src/config/aws.rs index c308607d0..9f53c328c 100644 --- a/htsget-config/src/config/aws.rs +++ b/htsget-config/src/config/aws.rs @@ -1,9 +1,9 @@ use serde; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; /// Configuration for the htsget server. -#[derive(Deserialize, Debug, Clone, Default)] +#[derive(Deserialize, Serialize, Debug, Clone, Default)] #[serde(default)] pub struct AwsS3DataServer { pub bucket: String, -} \ No newline at end of file +} diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 8e8e9950d..edf9c5c04 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -3,14 +3,15 @@ use std::io::ErrorKind; use std::net::SocketAddr; use std::path::PathBuf; +use crate::config::aws::AwsS3DataServer; use clap::Parser; -use config::File; -use serde::Deserialize; +use figment::providers::{Env, Format, Serialized, Toml}; +use figment::Figment; +use serde::{Deserialize, Serialize}; use tracing::info; use tracing::instrument; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::{fmt, EnvFilter, Registry}; -use crate::config::aws::AwsS3DataServer; use crate::config::StorageType::LocalStorage; use crate::regex_resolver::RegexResolver; @@ -55,7 +56,7 @@ The next variables are used to configure the info for the service-info endpoints * HTSGET_ENVIRONMENT: The environment in which the service is running. Default: "None". "#; -const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET"; +const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET_"; fn default_localstorage_addr() -> SocketAddr { "127.0.0.1:8081".parse().expect("expected valid address") @@ -90,7 +91,7 @@ pub struct Args { } /// Configuration for the server. Each field will be read from environment variables. -#[derive(Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct Config { pub ticket_server_config: TicketServerConfig, @@ -98,7 +99,7 @@ pub struct Config { } /// Configuration for the htsget server. -#[derive(Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { pub ticket_server_addr: SocketAddr, @@ -109,7 +110,7 @@ pub struct TicketServerConfig { } /// Configuration for the htsget server. -#[derive(Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct LocalDataServer { pub path: PathBuf, @@ -136,7 +137,7 @@ impl Default for LocalDataServer { } /// Specify the storage type to use. -#[derive(Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[non_exhaustive] pub enum StorageType { LocalStorage(LocalDataServer), @@ -151,7 +152,7 @@ impl Default for StorageType { } /// Configuration for the htsget server. -#[derive(Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct DataServerConfig { pub data_server_addr: SocketAddr, @@ -162,7 +163,7 @@ pub struct DataServerConfig { } /// Configuration of the service info. -#[derive(Deserialize, Debug, Clone, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct ServiceInfo { pub id: Option, @@ -218,23 +219,16 @@ impl Config { /// Read the environment variables into a Config struct. #[instrument] pub fn from_env(config: PathBuf) -> io::Result { - let config = config::Config::builder() - .add_source(File::from(config)) - .add_source(config::Environment::with_prefix( - ENVIRONMENT_VARIABLE_PREFIX, - )) - .build() + let config = Figment::from(Serialized::defaults(Config::default())) + .merge(Toml::file(config)) + .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX)) + .extract() .map_err(|err| { - io::Error::new( - ErrorKind::Other, - format!("config not properly set: {}", err), - ) - })? - .try_deserialize::() - .map_err(|err| io::Error::new(ErrorKind::Other, format!("failed to parse config: {}", err))); + io::Error::new(ErrorKind::Other, format!("failed to parse config: {}", err)) + })?; info!(config = ?config, "config created from environment variables"); - config + Ok(config) } /// Setup tracing, using a global subscriber. @@ -333,4 +327,4 @@ mod tests { // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); // assert_eq!(config.storage_type, StorageType::AwsS3Storage); // } -} \ No newline at end of file +} diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index b7cfb9f06..b81ca72d5 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -79,7 +79,7 @@ pub enum Class { /// An interval represents the start (0-based, inclusive) and end (0-based exclusive) ranges of the /// query. -#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize)] +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] pub struct Interval { pub start: Option, pub end: Option, @@ -150,7 +150,7 @@ impl Interval { } /// Possible values for the fields parameter. -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] pub enum Fields { /// Include all fields All, @@ -159,7 +159,7 @@ pub enum Fields { } /// Possible values for the tags parameter. -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] pub enum Tags { /// Include all tags All, @@ -168,7 +168,7 @@ pub enum Tags { } /// The no tags parameter. -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] pub struct NoTags(pub Option>); /// A query contains all the parameters that can be used when requesting diff --git a/htsget-config/src/regex_resolver.rs b/htsget-config/src/regex_resolver.rs index c6a0c0af7..df33d3c25 100644 --- a/htsget-config/src/regex_resolver.rs +++ b/htsget-config/src/regex_resolver.rs @@ -1,10 +1,10 @@ use regex::{Error, Regex}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tracing::instrument; +use crate::config::StorageType; use crate::Format::{Bam, Bcf, Cram, Vcf}; use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; -use crate::config::StorageType; /// Represents an id resolver, which matches the id, replacing the match in the substitution text. pub trait HtsGetIdResolver { @@ -19,7 +19,7 @@ pub trait QueryMatcher { } /// A regex resolver is a resolver that matches ids using Regex. -#[derive(Debug, Clone, Deserialize)] +#[derive(Serialize, Debug, Clone, Deserialize)] #[serde(default)] pub struct RegexResolver { #[serde(with = "serde_regex")] @@ -31,7 +31,7 @@ pub struct RegexResolver { } /// A query that can be matched with the regex resolver. -#[derive(Clone, Debug, Deserialize)] +#[derive(Serialize, Clone, Debug, Deserialize)] pub struct MatchOnQuery { pub format: Vec, pub class: Vec, @@ -111,13 +111,8 @@ impl QueryMatcher for MatchOnQuery { impl Default for RegexResolver { fn default() -> Self { - Self::new( - ".*", - "$0", - StorageType::default(), - MatchOnQuery::default(), - ) - .expect("expected valid resolver") + Self::new(".*", "$0", StorageType::default(), MatchOnQuery::default()) + .expect("expected valid resolver") } } From 69ee97a0a3605d768273eeb1f4b270913a1d5dbf Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 24 Nov 2022 10:58:41 +1100 Subject: [PATCH 10/45] refactor: fix errors relating to new config --- htsget-config/src/config/aws.rs | 2 ++ htsget-config/src/config/mod.rs | 44 ++++++++---------------- htsget-config/src/regex_resolver.rs | 7 ++++ htsget-http-actix/src/lib.rs | 10 +++--- htsget-http-actix/src/main.rs | 26 ++++++++------ htsget-http-core/src/lib.rs | 4 ++- htsget-http-lambda/src/lib.rs | 26 +++++++------- htsget-http-lambda/src/main.rs | 32 +++++++++-------- htsget-search/src/htsget/from_storage.rs | 4 +-- htsget-search/src/lib.rs | 4 ++- htsget-search/src/storage/aws.rs | 4 +-- htsget-search/src/storage/data_server.rs | 20 +++++------ htsget-search/src/storage/local.rs | 4 +-- htsget-test-utils/src/http_tests.rs | 35 +++++++++++-------- htsget-test-utils/src/lib.rs | 4 ++- htsget-test-utils/src/server_tests.rs | 17 ++++++--- 16 files changed, 135 insertions(+), 108 deletions(-) diff --git a/htsget-config/src/config/aws.rs b/htsget-config/src/config/aws.rs index 9f53c328c..9a5964453 100644 --- a/htsget-config/src/config/aws.rs +++ b/htsget-config/src/config/aws.rs @@ -6,4 +6,6 @@ use serde::{Deserialize, Serialize}; #[serde(default)] pub struct AwsS3DataServer { pub bucket: String, + pub cors_allow_credentials: bool, + pub cors_allow_origin: String, } diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index edf9c5c04..8950796e0 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -102,11 +102,11 @@ pub struct Config { #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { - pub ticket_server_addr: SocketAddr, + pub addr: SocketAddr, #[serde(flatten)] pub service_info: ServiceInfo, - pub ticket_server_cors_allow_credentials: bool, - pub ticket_server_cors_allow_origin: String, + pub cors_allow_credentials: bool, + pub cors_allow_origin: String, } /// Configuration for the htsget server. @@ -151,17 +151,6 @@ impl Default for StorageType { } } -/// Configuration for the htsget server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct DataServerConfig { - pub data_server_addr: SocketAddr, - pub data_server_key: Option, - pub data_server_cert: Option, - pub data_server_cors_allow_credentials: bool, - pub data_server_cors_allow_origin: String, -} - /// Configuration of the service info. #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] @@ -181,31 +170,28 @@ pub struct ServiceInfo { impl Default for TicketServerConfig { fn default() -> Self { Self { - ticket_server_addr: default_addr(), + addr: default_addr(), service_info: ServiceInfo::default(), - ticket_server_cors_allow_credentials: false, - ticket_server_cors_allow_origin: default_ticket_server_origin(), + cors_allow_credentials: false, + cors_allow_origin: default_ticket_server_origin(), } } } -impl Default for DataServerConfig { +impl Default for Config { fn default() -> Self { Self { - data_server_addr: default_localstorage_addr(), - data_server_key: None, - data_server_cert: None, - data_server_cors_allow_credentials: false, - data_server_cors_allow_origin: default_data_server_origin(), + resolvers: vec![RegexResolver::default()], + ticket_server_config: Default::default(), } } } -impl Default for Config { - fn default() -> Self { +impl From for Config { + fn from(config: LocalDataServer) -> Self { Self { - resolvers: vec![RegexResolver::default()], - ticket_server_config: Default::default(), + resolvers: vec![RegexResolver::from(LocalStorage(config))], + ticket_server_config: Default::default() } } } @@ -258,7 +244,7 @@ mod tests { std::env::set_var("HTSGET_TICKET_SERVER_ADDR", "127.0.0.1:8081"); let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); assert_eq!( - config.ticket_server_config.ticket_server_addr, + config.ticket_server_config.addr, "127.0.0.1:8081".parse().unwrap() ); } @@ -271,7 +257,7 @@ mod tests { ); let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); assert_eq!( - config.ticket_server_config.ticket_server_cors_allow_origin, + config.ticket_server_config.cors_allow_origin, "http://localhost:8080" ); } diff --git a/htsget-config/src/regex_resolver.rs b/htsget-config/src/regex_resolver.rs index df33d3c25..213d4dd6b 100644 --- a/htsget-config/src/regex_resolver.rs +++ b/htsget-config/src/regex_resolver.rs @@ -116,6 +116,13 @@ impl Default for RegexResolver { } } +impl From for RegexResolver { + fn from(storage_type: StorageType) -> Self { + Self::new(".*", "$0", storage_type, MatchOnQuery::default()) + .expect("expected valid resolver") + } +} + impl RegexResolver { /// Create a new regex resolver. pub fn new( diff --git a/htsget-http-actix/src/lib.rs b/htsget-http-actix/src/lib.rs index a001be86b..574a68c03 100644 --- a/htsget-http-actix/src/lib.rs +++ b/htsget-http-actix/src/lib.rs @@ -7,8 +7,10 @@ use tracing::info; use tracing::instrument; use tracing_actix_web::TracingLogger; +#[cfg(feature = "s3-storage")] +pub use htsget_config::config::aws::AwsS3DataServer; pub use htsget_config::config::{ - Config, DataServerConfig, ServiceInfo, StorageType, TicketServerConfig, USAGE, + Config, LocalDataServer, ServiceInfo, StorageType, TicketServerConfig, USAGE, }; use htsget_search::htsget::from_storage::HtsGetFromStorage; use htsget_search::htsget::HtsGet; @@ -84,12 +86,12 @@ pub fn run_server( configure_server(service_config, htsget.clone(), config.service_info.clone()); }) .wrap(configure_cors( - config.ticket_server_cors_allow_credentials, - config.ticket_server_cors_allow_origin.clone(), + config.cors_allow_credentials, + config.cors_allow_origin.clone(), )) .wrap(TracingLogger::default()) })) - .bind(config.ticket_server_addr)?; + .bind(config.addr)?; info!(addresses = ?server.addrs(), "htsget query server addresses bound"); Ok(server.run()) diff --git a/htsget-http-actix/src/main.rs b/htsget-http-actix/src/main.rs index 3c4e7ef1a..4aa55881a 100644 --- a/htsget-http-actix/src/main.rs +++ b/htsget-http-actix/src/main.rs @@ -1,6 +1,9 @@ use std::io::{Error, ErrorKind}; use tokio::select; +use htsget_config::config::aws::AwsS3DataServer; +use htsget_config::config::{LocalDataServer, TicketServerConfig}; +use htsget_config::regex_resolver::RegexResolver; use htsget_http_actix::run_server; use htsget_http_actix::{Config, StorageType}; @@ -12,33 +15,34 @@ async fn main() -> std::io::Result<()> { Config::setup_tracing()?; let config = Config::from_env(Config::parse_args())?; - match config.storage_type { - StorageType::LocalStorage => local_storage_server(config).await, + let resolver = config.resolvers.first().unwrap(); + match resolver.server.clone() { + StorageType::LocalStorage(server_config) => local_storage_server(server_config.clone(), resolver, config.ticket_server_config).await, #[cfg(feature = "s3-storage")] - StorageType::AwsS3Storage => s3_storage_server(config).await, + StorageType::AwsS3Storage(server_config) => s3_storage_server(&server_config, resolver, config.ticket_server_config).await, _ => Err(Error::new(ErrorKind::Other, "unsupported storage type")), } } -async fn local_storage_server(config: Config) -> std::io::Result<()> { - let mut formatter = HttpTicketFormatter::try_from(config.data_server_config)?; +async fn local_storage_server(config: LocalDataServer, resolver: &RegexResolver, ticket_config: TicketServerConfig) -> std::io::Result<()> { + let mut formatter = HttpTicketFormatter::try_from(config.clone())?; let local_server = formatter.bind_data_server().await?; let searcher = - HtsGetFromStorage::local_from(config.path.clone(), config.resolver.clone(), formatter)?; - let local_server = tokio::spawn(async move { local_server.serve(&config.path).await }); + HtsGetFromStorage::local_from(config.path.clone(), resolver.clone(), formatter)?; + let local_server = tokio::spawn(async move { local_server.serve(&config.path.clone()).await }); select! { local_server = local_server => Ok(local_server??), actix_server = run_server( searcher, - config.ticket_server_config, + ticket_config, )? => actix_server } } #[cfg(feature = "s3-storage")] -async fn s3_storage_server(config: Config) -> std::io::Result<()> { - let searcher = HtsGetFromStorage::s3_from(config.s3_bucket, config.resolver).await; - run_server(searcher, config.ticket_server_config)?.await +async fn s3_storage_server(config: &AwsS3DataServer, resolver: &RegexResolver, ticket_config: TicketServerConfig) -> std::io::Result<()> { + let searcher = HtsGetFromStorage::s3_from(config.bucket.clone(), resolver.clone()).await; + run_server(searcher, ticket_config)?.await } diff --git a/htsget-http-core/src/lib.rs b/htsget-http-core/src/lib.rs index b52caa73f..7106fa793 100644 --- a/htsget-http-core/src/lib.rs +++ b/htsget-http-core/src/lib.rs @@ -2,8 +2,10 @@ use std::collections::HashMap; use std::str::FromStr; pub use error::{HtsGetError, Result}; +#[cfg(feature = "s3-storage")] +pub use htsget_config::config::aws::AwsS3DataServer; pub use htsget_config::config::{ - Config, DataServerConfig, ServiceInfo as ConfigServiceInfo, StorageType, TicketServerConfig, + Config, LocalDataServer, ServiceInfo as ConfigServiceInfo, StorageType, TicketServerConfig, }; use htsget_config::Query; use htsget_search::htsget::Response; diff --git a/htsget-http-lambda/src/lib.rs b/htsget-http-lambda/src/lib.rs index d115fbe13..7a6c84367 100644 --- a/htsget-http-lambda/src/lib.rs +++ b/htsget-http-lambda/src/lib.rs @@ -12,8 +12,10 @@ use lambda_runtime::Error; use tracing::instrument; use tracing::{debug, info}; +#[cfg(feature = "s3-storage")] +pub use htsget_config::config::aws::AwsS3DataServer; pub use htsget_config::config::{ - Config, DataServerConfig, ServiceInfo, StorageType, TicketServerConfig, + Config, LocalDataServer, ServiceInfo, StorageType, TicketServerConfig, }; use htsget_http_core::{Endpoint, PostRequest}; use htsget_search::htsget::HtsGet; @@ -303,7 +305,7 @@ mod tests { let router = Router::new( Arc::new( - HtsGetFromStorage::local_from(&self.config.path, self.config.resolver.clone(), formatter) + HtsGetFromStorage::local_from(&self.config.resolvers.first().unwrap().server, self.config.resolver.clone(), formatter) .unwrap(), ), &self.config.ticket_server_config.service_info, @@ -494,7 +496,7 @@ mod tests { assert!(router.get_route(&Method::DELETE, &uri).is_none()); }, &config, - formatter_from_config(&config), + formatter_from_config(&config).unwrap(), ) .await; } @@ -508,7 +510,7 @@ mod tests { assert!(router.get_route(&Method::GET, &uri).is_none()); }, &config, - formatter_from_config(&config), + formatter_from_config(&config).unwrap(), ) .await; } @@ -522,7 +524,7 @@ mod tests { assert!(router.get_route(&Method::GET, &uri).is_none()); }, &config, - formatter_from_config(&config), + formatter_from_config(&config).unwrap(), ) .await; } @@ -536,7 +538,7 @@ mod tests { assert!(router.get_route(&Method::GET, &uri).is_none()); }, &config, - formatter_from_config(&config), + formatter_from_config(&config).unwrap(), ) .await; } @@ -550,7 +552,7 @@ mod tests { assert!(router.get_route(&Method::GET, &uri).is_none()); }, &config, - formatter_from_config(&config), + formatter_from_config(&config).unwrap(), ) .await; } @@ -575,7 +577,7 @@ mod tests { ); }, &config, - formatter_from_config(&config), + formatter_from_config(&config).unwrap(), ) .await; } @@ -600,7 +602,7 @@ mod tests { ); }, &config, - formatter_from_config(&config), + formatter_from_config(&config).unwrap(), ) .await; } @@ -622,7 +624,7 @@ mod tests { ); }, &config, - formatter_from_config(&config), + formatter_from_config(&config).unwrap(), ) .await; } @@ -647,7 +649,7 @@ mod tests { ); }, &config, - formatter_from_config(&config), + formatter_from_config(&config).unwrap(), ) .await; } @@ -691,7 +693,7 @@ mod tests { } async fn test_service_info_from_file(file_path: &str, config: &Config) { - let formatter = formatter_from_config(config); + let formatter = formatter_from_config(config).unwrap(); let expected_path = expected_url_path(&formatter); with_router( |router| async { diff --git a/htsget-http-lambda/src/main.rs b/htsget-http-lambda/src/main.rs index 48932c77e..894904f6e 100644 --- a/htsget-http-lambda/src/main.rs +++ b/htsget-http-lambda/src/main.rs @@ -2,6 +2,9 @@ use std::sync::Arc; use lambda_http::Error; use tracing::instrument; +use htsget_config::config::{LocalDataServer, TicketServerConfig}; +use htsget_config::config::aws::AwsS3DataServer; +use htsget_config::regex_resolver::RegexResolver; use htsget_http_lambda::{handle_request, Router}; use htsget_http_lambda::{Config, StorageType}; @@ -14,25 +17,26 @@ async fn main() -> Result<(), Error> { Config::setup_tracing()?; let config = Config::from_env(Config::parse_args())?; - match config.storage_type { - StorageType::LocalStorage => local_storage_server(config).await, + let resolver = config.resolvers.first().unwrap(); + match resolver.server.clone() { + StorageType::LocalStorage(server_config) => local_storage_server(&server_config, resolver, config.ticket_server_config).await, #[cfg(feature = "s3-storage")] - StorageType::AwsS3Storage => s3_storage_server(config).await, + StorageType::AwsS3Storage(server_config) => s3_storage_server(&server_config, resolver, config.ticket_server_config).await, _ => Err("unsupported storage type".into()), } } #[instrument(skip_all)] -async fn local_storage_server(config: Config) -> Result<(), Error> { - let formatter = HttpTicketFormatter::try_from(config.data_server_config.clone())?; +async fn local_storage_server(config: &LocalDataServer, resolver: &RegexResolver, ticket_config: TicketServerConfig) -> Result<(), Error> { + let formatter = HttpTicketFormatter::try_from(config.clone())?; let searcher: Arc>> = Arc::new( - HtsGetFromStorage::local_from(config.path, config.resolver, formatter)?, + HtsGetFromStorage::local_from(config.path.clone(), resolver.clone(), formatter)?, ); - let router = &Router::new(searcher, &config.ticket_server_config.service_info); + let router = &Router::new(searcher, &ticket_config.service_info); handle_request( - config.data_server_config.data_server_cors_allow_credentials, - config.data_server_config.data_server_cors_allow_origin, + config.cors_allow_credentials, + config.cors_allow_origin.clone(), router, ) .await @@ -40,13 +44,13 @@ async fn local_storage_server(config: Config) -> Result<(), Error> { #[cfg(feature = "s3-storage")] #[instrument(skip_all)] -async fn s3_storage_server(config: Config) -> Result<(), Error> { - let searcher = Arc::new(HtsGetFromStorage::s3_from(config.s3_bucket, config.resolver).await); - let router = &Router::new(searcher, &config.ticket_server_config.service_info); +async fn s3_storage_server(config: &AwsS3DataServer, resolver: &RegexResolver, ticket_config: TicketServerConfig) -> Result<(), Error> { + let searcher = Arc::new(HtsGetFromStorage::s3_from(config.bucket.clone(), resolver.clone()).await); + let router = &Router::new(searcher, &ticket_config.service_info); handle_request( - config.data_server_config.data_server_cors_allow_credentials, - config.data_server_config.data_server_cors_allow_origin, + config.cors_allow_credentials, + config.cors_allow_origin.clone(), router, ) .await diff --git a/htsget-search/src/htsget/from_storage.rs b/htsget-search/src/htsget/from_storage.rs index f597153a3..62549293a 100644 --- a/htsget-search/src/htsget/from_storage.rs +++ b/htsget-search/src/htsget/from_storage.rs @@ -100,7 +100,7 @@ pub(crate) mod tests { use tempfile::TempDir; - use htsget_config::config::StorageTypeServer; + use htsget_config::config::StorageType; use htsget_config::regex_resolver::MatchOnQuery; use htsget_test_utils::util::expected_bgzf_eof_data_url; @@ -183,7 +183,7 @@ pub(crate) mod tests { RegexResolver::new( ".*", "$0", - StorageTypeServer::default(), + StorageType::default(), MatchOnQuery::default(), ) .unwrap(), diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index 25917eee1..a348c4983 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -1,5 +1,7 @@ +#[cfg(feature = "s3-storage")] +pub use htsget_config::config::aws::AwsS3DataServer; pub use htsget_config::config::{ - Config, DataServerConfig, ServiceInfo, StorageType, TicketServerConfig, + Config, LocalDataServer, ServiceInfo, StorageType, TicketServerConfig, }; pub use htsget_config::regex_resolver::{HtsGetIdResolver, RegexResolver}; diff --git a/htsget-search/src/storage/aws.rs b/htsget-search/src/storage/aws.rs index b196b1305..6f0e527d6 100644 --- a/htsget-search/src/storage/aws.rs +++ b/htsget-search/src/storage/aws.rs @@ -225,7 +225,7 @@ mod tests { use s3_server::storages::fs::FileSystem; use s3_server::{S3Service, SimpleAuth}; - use htsget_config::config::StorageTypeServer; + use htsget_config::config::StorageType; use htsget_config::regex_resolver::MatchOnQuery; use htsget_config::Format::Bam; use htsget_config::Query; @@ -287,7 +287,7 @@ mod tests { RegexResolver::new( ".*", "$0", - StorageTypeServer::default(), + StorageType::default(), MatchOnQuery::default(), ) .unwrap(), diff --git a/htsget-search/src/storage/data_server.rs b/htsget-search/src/storage/data_server.rs index 49692f958..05f632fd3 100644 --- a/htsget-search/src/storage/data_server.rs +++ b/htsget-search/src/storage/data_server.rs @@ -26,10 +26,10 @@ use tower::MakeService; use tower_http::trace::TraceLayer; use tracing::instrument; use tracing::{info, trace}; +use htsget_config::config::LocalDataServer; use crate::storage::StorageError::{DataServerError, IoError}; use crate::storage::{configure_cors, UrlFormatter}; -use crate::DataServerConfig; use super::{Result, StorageError}; @@ -112,17 +112,17 @@ impl HttpTicketFormatter { } } -impl TryFrom for HttpTicketFormatter { +impl TryFrom for HttpTicketFormatter { type Error = StorageError; /// Returns a ticket server with tls if both cert and key are not None, without tls if cert and key /// are both None, and otherwise an error. - fn try_from(config: DataServerConfig) -> Result { - match (config.data_server_cert, config.data_server_key) { + fn try_from(config: LocalDataServer) -> Result { + match (config.cert, config.key) { (Some(cert), Some(key)) => Ok(Self::new_with_tls( - config.data_server_addr, - config.data_server_cors_allow_origin, - config.data_server_cors_allow_credentials, + config.addr, + config.cors_allow_origin, + config.cors_allow_credentials, cert, key, )), @@ -130,9 +130,9 @@ impl TryFrom for HttpTicketFormatter { "both the cert and key must be provided for the ticket server".to_string(), )), (None, None) => Ok(Self::new( - config.data_server_addr, - config.data_server_cors_allow_origin, - config.data_server_cors_allow_credentials, + config.addr, + config.cors_allow_origin, + config.cors_allow_credentials, )), } } diff --git a/htsget-search/src/storage/local.rs b/htsget-search/src/storage/local.rs index 516d54f9e..577c1644a 100644 --- a/htsget-search/src/storage/local.rs +++ b/htsget-search/src/storage/local.rs @@ -126,7 +126,7 @@ pub(crate) mod tests { use tokio::fs::{create_dir, File}; use tokio::io::AsyncWriteExt; - use htsget_config::config::StorageTypeServer; + use htsget_config::config::StorageType; use htsget_config::regex_resolver::MatchOnQuery; use htsget_config::Format::Bam; @@ -324,7 +324,7 @@ pub(crate) mod tests { RegexResolver::new( ".*", "$0", - StorageTypeServer::default(), + StorageType::default(), MatchOnQuery::default(), ) .unwrap(), diff --git a/htsget-test-utils/src/http_tests.rs b/htsget-test-utils/src/http_tests.rs index 93b5562ed..2e204f0f2 100644 --- a/htsget-test-utils/src/http_tests.rs +++ b/htsget-test-utils/src/http_tests.rs @@ -4,6 +4,8 @@ use std::path::{Path, PathBuf}; use async_trait::async_trait; use http::HeaderMap; use serde::de; +use htsget_config::config::{LocalDataServer, StorageType}; +use htsget_config::regex_resolver::RegexResolver; use crate::util::generate_test_certificates; use crate::Config; @@ -83,43 +85,46 @@ pub fn default_dir_data() -> PathBuf { default_dir().join("data") } -fn set_path(config: &mut Config) { +fn set_path(config: &mut LocalDataServer) { config.path = default_dir_data(); } -fn set_addr_and_path(config: &mut Config) { +fn set_addr_and_path(config: &mut LocalDataServer) { set_path(config); - config.data_server_config.data_server_addr = "127.0.0.1:0".parse().unwrap(); + config.addr = "127.0.0.1:0".parse().unwrap(); } /// Default config with fixed port. -pub fn default_config_fixed_port() -> Config { - let mut config = Config::default(); +pub fn default_config_fixed_port() -> LocalDataServer { + let mut config = LocalDataServer::default(); set_path(&mut config); config } /// Default config using the current cargo manifest directory, and dynamic port. pub fn default_test_config() -> Config { - let mut config = Config::default(); - set_addr_and_path(&mut config); + let mut server_config = LocalDataServer::default(); + set_addr_and_path(&mut server_config); - config.data_server_config.data_server_cors_allow_credentials = false; - config.data_server_config.data_server_cors_allow_origin = "http://example.com".to_string(); + let mut server_config = LocalDataServer::default(); + server_config.cors_allow_credentials = false; + server_config.cors_allow_origin = "http://example.com".to_string(); - config + Config::from(server_config) } /// Config with tls ticket server, using the current cargo manifest directory. pub fn config_with_tls>(path: P) -> Config { - let mut config = Config::default(); - set_addr_and_path(&mut config); + let mut server_config = LocalDataServer::default(); + set_addr_and_path(&mut server_config); let (key_path, cert_path) = generate_test_certificates(path, "key.pem", "cert.pem"); - config.data_server_config.data_server_key = Some(key_path); - config.data_server_config.data_server_cert = Some(cert_path); - config + let mut server_config = LocalDataServer::default(); + server_config.key = Some(key_path); + server_config.cert = Some(cert_path); + + Config::from(server_config) } /// Get the event associated with the file. diff --git a/htsget-test-utils/src/lib.rs b/htsget-test-utils/src/lib.rs index 99e2c87bc..2dcd93bd6 100644 --- a/htsget-test-utils/src/lib.rs +++ b/htsget-test-utils/src/lib.rs @@ -1,6 +1,8 @@ +#[cfg(feature = "s3-storage")] +pub use htsget_config::config::aws::AwsS3DataServer; #[cfg(any(feature = "cors-tests", feature = "server-tests"))] pub use htsget_config::config::{ - Config, DataServerConfig, ServiceInfo, StorageType, TicketServerConfig, + Config, LocalDataServer, ServiceInfo, StorageType, TicketServerConfig, }; #[cfg(feature = "cors-tests")] diff --git a/htsget-test-utils/src/server_tests.rs b/htsget-test-utils/src/server_tests.rs index 16a84fd49..008c7b9da 100644 --- a/htsget-test-utils/src/server_tests.rs +++ b/htsget-test-utils/src/server_tests.rs @@ -8,6 +8,7 @@ use http::Method; use noodles_bgzf as bgzf; use noodles_vcf as vcf; use reqwest::ClientBuilder; +use htsget_config::config::StorageType::LocalStorage; use htsget_http_core::{get_service_info_with, Endpoint}; use htsget_search::htsget::Response as HtsgetResponse; @@ -73,8 +74,12 @@ pub async fn test_response(response: Response, class: Class) { /// Create the a [HttpTicketFormatter], spawn the ticket server, returning the expected path and the formatter. pub async fn formatter_and_expected_path(config: &Config) -> (String, HttpTicketFormatter) { - let mut formatter = formatter_from_config(config); - spawn_ticket_server(config.path.clone(), &mut formatter).await; + let mut formatter = formatter_from_config(config).unwrap(); + for resolver in config.resolvers.iter() { + if let LocalStorage(server) = &resolver.server { + spawn_ticket_server(server.path.clone(), &mut formatter).await; + } + } (expected_url_path(&formatter), formatter) } @@ -162,8 +167,12 @@ pub async fn test_parameterized_post_class_header(tester: &impl } /// Get the [HttpTicketFormatter] from the config. -pub fn formatter_from_config(config: &Config) -> HttpTicketFormatter { - HttpTicketFormatter::try_from(config.data_server_config.clone()).unwrap() +pub fn formatter_from_config(config: &Config) -> Option { + if let LocalStorage(server_config) = config.resolvers.first()?.server.clone() { + HttpTicketFormatter::try_from(server_config).ok() + } else { + None + } } /// A service info test. From 52b7df1a39571e0384d884adc1b85d1896323462 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 5 Dec 2022 19:30:33 +1100 Subject: [PATCH 11/45] config: add UrlResolver, separate data server config from resolver --- htsget-config/Cargo.toml | 5 +- htsget-config/config.toml | 25 ++ htsget-config/src/config.rs | 294 ++++++++++++++++ htsget-config/src/config/mod.rs | 316 ------------------ htsget-config/src/lib.rs | 2 + htsget-config/src/regex_resolver.rs | 177 ---------- .../src/{config => regex_resolver}/aws.rs | 2 +- htsget-config/src/regex_resolver/mod.rs | 224 +++++++++++++ 8 files changed, 550 insertions(+), 495 deletions(-) create mode 100644 htsget-config/src/config.rs delete mode 100644 htsget-config/src/config/mod.rs delete mode 100644 htsget-config/src/regex_resolver.rs rename htsget-config/src/{config => regex_resolver}/aws.rs (89%) create mode 100644 htsget-config/src/regex_resolver/mod.rs diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index f136b2d60..779a14d1f 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -16,4 +16,7 @@ regex = "1.6" figment = { version = "0.10", features = ["env", "toml"] } clap = { version = "4.0", features = ["derive", "env"] } tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] } \ No newline at end of file +tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] } +toml = "0.5" +http = "0.2" +http-serde = "1.1" \ No newline at end of file diff --git a/htsget-config/config.toml b/htsget-config/config.toml index e69de29bb..3c4262631 100644 --- a/htsget-config/config.toml +++ b/htsget-config/config.toml @@ -0,0 +1,25 @@ +ticket_server_addr = "127.0.0.1:8080" +ticket_server_cors_allow_credentials = false +ticket_server_cors_allow_origin = "http://localhost:8080" +start_data_server = true +data_server_path = "data" +data_server_serve_at = "/data" +data_server_addr = "127.0.0.1:8081" +data_server_cors_allow_credentials = false +data_server_cors_allow_origin = "http://localhost:8081" + +[[resolver]] +regex = ".*" +substitution_string = "$0" + +storage_type.type = "Url" +storage_type.scheme = "Https" +storage_type.authority = "127.0.0.1:8081" +storage_type.path = "/data" + +[resolver.guard] +match_formats = ["BAM"] +start_interval.start = 100 +match_fields = ["field1"] +match_no_tags = ["tag1"] + diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs new file mode 100644 index 000000000..3329df2d8 --- /dev/null +++ b/htsget-config/src/config.rs @@ -0,0 +1,294 @@ +use std::io; +use std::io::ErrorKind; +use std::net::SocketAddr; +use std::path::PathBuf; + +use crate::regex_resolver::aws::S3Resolver; +use clap::Parser; +use figment::providers::{Env, Format, Serialized, Toml}; +use figment::Figment; +use serde::{Deserialize, Serialize}; +use tracing::info; +use tracing::instrument; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::{EnvFilter, fmt, Registry}; + +use crate::regex_resolver::RegexResolver; + +/// Represents a usage string for htsget-rs. +pub const USAGE: &str = r#" +Available environment variables: +* HTSGET_PATH: The path to the directory where the server should be started. Default: "data". Unused if HTSGET_STORAGE_TYPE is "AwsS3Storage". +* HTSGET_REGEX: The regular expression that should match an ID. Default: ".*". +For more information about the regex options look in the documentation of the regex crate(https://docs.rs/regex/). +* HTSGET_SUBSTITUTION_STRING: The replacement expression. Default: "$0". +* HTSGET_STORAGE_TYPE: Either "LocalStorage" or "AwsS3Storage", representing which storage type to use. Default: "LocalStorage". + +The following options are used for the ticket server. +* HTSGET_TICKET_SERVER_ADDR: The socket address for the server which creates response tickets. Default: "127.0.0.1:8080". +* HTSGET_TICKET_SERVER_ALLOW_CREDENTIALS: Boolean flag, indicating whether authenticated requests are allowed by including the `Access-Control-Allow-Credentials` header. Default: "false". +* HTSGET_TICKET_SERVER_ALLOW_ORIGIN: Which origin os allowed in the `ORIGIN` header. Default: "http://localhost:8080". + +The following options are used for the data server. +* HTSGET_DATA_SERVER_ADDR: The socket address to use for the server which responds to tickets. Default: "127.0.0.1:8081". Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". +* HTSGET_DATA_SERVER_KEY: The path to the PEM formatted X.509 private key used by the data server. Default: "None". Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". +* HTSGET_DATA_SERVER_CERT: The path to the PEM formatted X.509 certificate used by the data server. Default: "None". Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". +* HTSGET_DATA_SERVER_ALLOW_CREDENTIALS: Boolean flag, indicating whether authenticated requests are allowed by including the `Access-Control-Allow-Credentials` header. Default: "false" +* HTSGET_DATA_SERVER_ALLOW_ORIGIN: Which origin os allowed in the `ORIGIN` header. Default: "http://localhost:8081" + +The following options are used to configure AWS S3 storage. +* HTSGET_S3_BUCKET: The name of the AWS S3 bucket. Default: "". Unused if HTSGET_STORAGE_TYPE is not "AwsS3Storage". + +The next variables are used to configure the info for the service-info endpoints. +* HTSGET_ID: The id of the service. Default: "None". +* HTSGET_NAME: The name of the service. Default: "None". +* HTSGET_VERSION: The version of the service. Default: "None". +* HTSGET_ORGANIZATION_NAME: The name of the organization. Default: "None". +* HTSGET_ORGANIZATION_URL: The url of the organization. Default: "None". +* HTSGET_CONTACT_URL: A url to provide contact to the users. Default: "None". +* HTSGET_DOCUMENTATION_URL: A link to the documentation. Default: "None". +* HTSGET_CREATED_AT: Date of the creation of the service. Default: "None". +* HTSGET_UPDATED_AT: Date of the last update of the service. Default: "None". +* HTSGET_ENVIRONMENT: The environment in which the service is running. Default: "None". +"#; + +const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET_"; + +pub(crate) fn default_localstorage_addr() -> &'static str { + "127.0.0.1:8081" +} + +fn default_addr() -> &'static str { + "127.0.0.1:8080" +} + +fn default_ticket_server_origin() -> &'static str { + "http://localhost:8080" +} + +fn default_data_server_origin() -> &'static str { + "http://localhost:8081" +} + +fn default_path() -> &'static str { + "data" +} + +pub(crate) fn default_serve_at() -> &'static str { + "/data" +} + +/// The command line arguments allowed for the htsget-rs executables. +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = USAGE)] +pub struct Args { + #[arg(short, long, env = "HTSGET_CONFIG")] + config: PathBuf, +} + +/// Configuration for the server. Each field will be read from environment variables. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct Config { + #[serde(flatten)] + pub ticket_server_config: TicketServerConfig, + #[serde(flatten)] + pub data_server_config: DataServerConfig, + pub resolver: Vec, +} + +/// Configuration for the htsget server. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct TicketServerConfig { + pub ticket_server_addr: SocketAddr, + pub ticket_server_cors_allow_credentials: bool, + pub ticket_server_cors_allow_origin: String, + #[serde(flatten)] + pub service_info: ServiceInfo, +} + +/// Configuration for the htsget server. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct DataServerConfig { + pub start_data_server: bool, + pub data_server_path: PathBuf, + pub data_server_serve_at: PathBuf, + pub data_server_addr: SocketAddr, + pub data_server_key: Option, + pub data_server_cert: Option, + pub data_server_cors_allow_credentials: bool, + pub data_server_cors_allow_origin: String, +} + +impl Default for DataServerConfig { + fn default() -> Self { + Self { + start_data_server: true, + data_server_path: default_path().into(), + data_server_serve_at: default_serve_at().into(), + data_server_addr: default_localstorage_addr().parse().expect("expected valid address"), + data_server_key: None, + data_server_cert: None, + data_server_cors_allow_credentials: false, + data_server_cors_allow_origin: default_data_server_origin().to_string(), + } + } +} + +/// Configuration of the service info. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct ServiceInfo { + pub id: Option, + pub name: Option, + pub version: Option, + pub organization_name: Option, + pub organization_url: Option, + pub contact_url: Option, + pub documentation_url: Option, + pub created_at: Option, + pub updated_at: Option, + pub environment: Option, +} + +impl Default for TicketServerConfig { + fn default() -> Self { + Self { + ticket_server_addr: default_addr().parse().expect("expected valid address"), + ticket_server_cors_allow_credentials: false, + ticket_server_cors_allow_origin: default_ticket_server_origin().to_string(), + service_info: ServiceInfo::default(), + } + } +} + +impl Default for Config { + fn default() -> Self { + Self { + ticket_server_config: Default::default(), + data_server_config: Default::default(), + resolver: vec![RegexResolver::default(), RegexResolver::default()], + } + } +} + +impl Config { + /// Parse the command line arguments + pub fn parse_args() -> PathBuf { + Args::parse().config + } + + /// Read the environment variables into a Config struct. + #[instrument] + pub fn from_env(config: PathBuf) -> io::Result { + let config = Figment::from(Serialized::defaults(Config::default())) + .merge(Toml::file(config)) + .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX)) + .extract() + .map_err(|err| { + io::Error::new(ErrorKind::Other, format!("failed to parse config: {}", err)) + })?; + + info!(config = ?config, "config created from environment variables"); + Ok(config) + } + + /// Setup tracing, using a global subscriber. + pub fn setup_tracing() -> io::Result<()> { + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + let fmt_layer = fmt::Layer::default(); + + let subscriber = Registry::default().with(env_filter).with(fmt_layer); + + tracing::subscriber::set_global_default(subscriber).map_err(|err| { + io::Error::new( + ErrorKind::Other, + format!("failed to install `tracing` subscriber: {}", err), + ) + })?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // #[test] + // fn config_addr() { + // std::env::set_var("HTSGET_TICKET_SERVER_ADDR", "127.0.0.1:8081"); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!( + // config.ticket_server_config.addr, + // "127.0.0.1:8081".parse().unwrap() + // ); + // } + + // #[test] + // fn config_ticket_server_cors_allow_origin() { + // std::env::set_var( + // "HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGIN", + // "http://localhost:8080", + // ); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!( + // config.ticket_server_config.cors_allow_origin, + // "http://localhost:8080" + // ); + // } + + // #[test] + // fn config_data_server_cors_allow_origin() { + // std::env::set_var( + // "HTSGET_DATA_SERVER_CORS_ALLOW_ORIGIN", + // "http://localhost:8080", + // ); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!( + // config.data_server_config.data_server_cors_allow_origin, + // "http://localhost:8080" + // ); + // } + // + // #[test] + // fn config_ticket_server_addr() { + // std::env::set_var("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082"); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!( + // config.data_server_config.data_server_addr, + // "127.0.0.1:8082".parse().unwrap() + // ); + // } + // + // #[test] + // fn config_regex() { + // std::env::set_var("HTSGET_REGEX", ".+"); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!(config.resolver.regex.to_string(), ".+"); + // } + // + // #[test] + // fn config_substitution_string() { + // std::env::set_var("HTSGET_SUBSTITUTION_STRING", "$0-test"); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!(config.resolver.substitution_string, "$0-test"); + // } + + #[test] + fn config_service_info_id() { + std::env::set_var("HTSGET_ID", "id"); + let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + assert_eq!(config.ticket_server_config.service_info.id.unwrap(), "id"); + } + + // #[cfg(feature = "s3-storage")] + // #[test] + // fn config_storage_type() { + // std::env::set_var("HTSGET_STORAGE_TYPE", "AwsS3Storage"); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!(config.storage_type, StorageType::AwsS3Storage); + // } +} diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs deleted file mode 100644 index 8950796e0..000000000 --- a/htsget-config/src/config/mod.rs +++ /dev/null @@ -1,316 +0,0 @@ -use std::io; -use std::io::ErrorKind; -use std::net::SocketAddr; -use std::path::PathBuf; - -use crate::config::aws::AwsS3DataServer; -use clap::Parser; -use figment::providers::{Env, Format, Serialized, Toml}; -use figment::Figment; -use serde::{Deserialize, Serialize}; -use tracing::info; -use tracing::instrument; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::{fmt, EnvFilter, Registry}; - -use crate::config::StorageType::LocalStorage; -use crate::regex_resolver::RegexResolver; - -#[cfg(feature = "s3-storage")] -pub mod aws; - -/// Represents a usage string for htsget-rs. -pub const USAGE: &str = r#" -Available environment variables: -* HTSGET_PATH: The path to the directory where the server should be started. Default: "data". Unused if HTSGET_STORAGE_TYPE is "AwsS3Storage". -* HTSGET_REGEX: The regular expression that should match an ID. Default: ".*". -For more information about the regex options look in the documentation of the regex crate(https://docs.rs/regex/). -* HTSGET_SUBSTITUTION_STRING: The replacement expression. Default: "$0". -* HTSGET_STORAGE_TYPE: Either "LocalStorage" or "AwsS3Storage", representing which storage type to use. Default: "LocalStorage". - -The following options are used for the ticket server. -* HTSGET_TICKET_SERVER_ADDR: The socket address for the server which creates response tickets. Default: "127.0.0.1:8080". -* HTSGET_TICKET_SERVER_ALLOW_CREDENTIALS: Boolean flag, indicating whether authenticated requests are allowed by including the `Access-Control-Allow-Credentials` header. Default: "false". -* HTSGET_TICKET_SERVER_ALLOW_ORIGIN: Which origin os allowed in the `ORIGIN` header. Default: "http://localhost:8080". - -The following options are used for the data server. -* HTSGET_DATA_SERVER_ADDR: The socket address to use for the server which responds to tickets. Default: "127.0.0.1:8081". Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". -* HTSGET_DATA_SERVER_KEY: The path to the PEM formatted X.509 private key used by the data server. Default: "None". Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". -* HTSGET_DATA_SERVER_CERT: The path to the PEM formatted X.509 certificate used by the data server. Default: "None". Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". -* HTSGET_DATA_SERVER_ALLOW_CREDENTIALS: Boolean flag, indicating whether authenticated requests are allowed by including the `Access-Control-Allow-Credentials` header. Default: "false" -* HTSGET_DATA_SERVER_ALLOW_ORIGIN: Which origin os allowed in the `ORIGIN` header. Default: "http://localhost:8081" - -The following options are used to configure AWS S3 storage. -* HTSGET_S3_BUCKET: The name of the AWS S3 bucket. Default: "". Unused if HTSGET_STORAGE_TYPE is not "AwsS3Storage". - -The next variables are used to configure the info for the service-info endpoints. -* HTSGET_ID: The id of the service. Default: "None". -* HTSGET_NAME: The name of the service. Default: "None". -* HTSGET_VERSION: The version of the service. Default: "None". -* HTSGET_ORGANIZATION_NAME: The name of the organization. Default: "None". -* HTSGET_ORGANIZATION_URL: The url of the organization. Default: "None". -* HTSGET_CONTACT_URL: A url to provide contact to the users. Default: "None". -* HTSGET_DOCUMENTATION_URL: A link to the documentation. Default: "None". -* HTSGET_CREATED_AT: Date of the creation of the service. Default: "None". -* HTSGET_UPDATED_AT: Date of the last update of the service. Default: "None". -* HTSGET_ENVIRONMENT: The environment in which the service is running. Default: "None". -"#; - -const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET_"; - -fn default_localstorage_addr() -> SocketAddr { - "127.0.0.1:8081".parse().expect("expected valid address") -} - -fn default_addr() -> SocketAddr { - "127.0.0.1:8080".parse().expect("expected valid address") -} - -fn default_ticket_server_origin() -> String { - "http://localhost:8080".to_string() -} - -fn default_data_server_origin() -> String { - "http://localhost:8081".to_string() -} - -fn default_path() -> PathBuf { - PathBuf::from("data") -} - -fn default_serve_at() -> PathBuf { - PathBuf::from("/data") -} - -/// The command line arguments allowed for the htsget-rs executables. -#[derive(Parser, Debug)] -#[command(author, version, about, long_about = USAGE)] -pub struct Args { - #[arg(short, long, env = "HTSGET_CONFIG")] - config: PathBuf, -} - -/// Configuration for the server. Each field will be read from environment variables. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct Config { - pub ticket_server_config: TicketServerConfig, - pub resolvers: Vec, -} - -/// Configuration for the htsget server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct TicketServerConfig { - pub addr: SocketAddr, - #[serde(flatten)] - pub service_info: ServiceInfo, - pub cors_allow_credentials: bool, - pub cors_allow_origin: String, -} - -/// Configuration for the htsget server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct LocalDataServer { - pub path: PathBuf, - pub serve_at: PathBuf, - pub addr: SocketAddr, - pub key: Option, - pub cert: Option, - pub cors_allow_credentials: bool, - pub cors_allow_origin: String, -} - -impl Default for LocalDataServer { - fn default() -> Self { - Self { - path: default_path(), - serve_at: default_serve_at(), - addr: default_localstorage_addr(), - key: None, - cert: None, - cors_allow_credentials: false, - cors_allow_origin: default_data_server_origin(), - } - } -} - -/// Specify the storage type to use. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[non_exhaustive] -pub enum StorageType { - LocalStorage(LocalDataServer), - #[cfg(feature = "s3-storage")] - AwsS3Storage(AwsS3DataServer), -} - -impl Default for StorageType { - fn default() -> Self { - LocalStorage(LocalDataServer::default()) - } -} - -/// Configuration of the service info. -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct ServiceInfo { - pub id: Option, - pub name: Option, - pub version: Option, - pub organization_name: Option, - pub organization_url: Option, - pub contact_url: Option, - pub documentation_url: Option, - pub created_at: Option, - pub updated_at: Option, - pub environment: Option, -} - -impl Default for TicketServerConfig { - fn default() -> Self { - Self { - addr: default_addr(), - service_info: ServiceInfo::default(), - cors_allow_credentials: false, - cors_allow_origin: default_ticket_server_origin(), - } - } -} - -impl Default for Config { - fn default() -> Self { - Self { - resolvers: vec![RegexResolver::default()], - ticket_server_config: Default::default(), - } - } -} - -impl From for Config { - fn from(config: LocalDataServer) -> Self { - Self { - resolvers: vec![RegexResolver::from(LocalStorage(config))], - ticket_server_config: Default::default() - } - } -} - -impl Config { - /// Parse the command line arguments - pub fn parse_args() -> PathBuf { - Args::parse().config - } - - /// Read the environment variables into a Config struct. - #[instrument] - pub fn from_env(config: PathBuf) -> io::Result { - let config = Figment::from(Serialized::defaults(Config::default())) - .merge(Toml::file(config)) - .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX)) - .extract() - .map_err(|err| { - io::Error::new(ErrorKind::Other, format!("failed to parse config: {}", err)) - })?; - - info!(config = ?config, "config created from environment variables"); - Ok(config) - } - - /// Setup tracing, using a global subscriber. - pub fn setup_tracing() -> io::Result<()> { - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - let fmt_layer = fmt::Layer::default(); - - let subscriber = Registry::default().with(env_filter).with(fmt_layer); - - tracing::subscriber::set_global_default(subscriber).map_err(|err| { - io::Error::new( - ErrorKind::Other, - format!("failed to install `tracing` subscriber: {}", err), - ) - })?; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn config_addr() { - std::env::set_var("HTSGET_TICKET_SERVER_ADDR", "127.0.0.1:8081"); - let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - assert_eq!( - config.ticket_server_config.addr, - "127.0.0.1:8081".parse().unwrap() - ); - } - - #[test] - fn config_ticket_server_cors_allow_origin() { - std::env::set_var( - "HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGIN", - "http://localhost:8080", - ); - let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - assert_eq!( - config.ticket_server_config.cors_allow_origin, - "http://localhost:8080" - ); - } - - // #[test] - // fn config_data_server_cors_allow_origin() { - // std::env::set_var( - // "HTSGET_DATA_SERVER_CORS_ALLOW_ORIGIN", - // "http://localhost:8080", - // ); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!( - // config.data_server_config.data_server_cors_allow_origin, - // "http://localhost:8080" - // ); - // } - // - // #[test] - // fn config_ticket_server_addr() { - // std::env::set_var("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082"); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!( - // config.data_server_config.data_server_addr, - // "127.0.0.1:8082".parse().unwrap() - // ); - // } - // - // #[test] - // fn config_regex() { - // std::env::set_var("HTSGET_REGEX", ".+"); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!(config.resolver.regex.to_string(), ".+"); - // } - // - // #[test] - // fn config_substitution_string() { - // std::env::set_var("HTSGET_SUBSTITUTION_STRING", "$0-test"); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!(config.resolver.substitution_string, "$0-test"); - // } - - #[test] - fn config_service_info_id() { - std::env::set_var("HTSGET_ID", "id"); - let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - assert_eq!(config.ticket_server_config.service_info.id.unwrap(), "id"); - } - - // #[cfg(feature = "s3-storage")] - // #[test] - // fn config_storage_type() { - // std::env::set_var("HTSGET_STORAGE_TYPE", "AwsS3Storage"); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!(config.storage_type, StorageType::AwsS3Storage); - // } -} diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index b81ca72d5..06ba8c4c0 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -151,6 +151,7 @@ impl Interval { /// Possible values for the fields parameter. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(untagged)] pub enum Fields { /// Include all fields All, @@ -160,6 +161,7 @@ pub enum Fields { /// Possible values for the tags parameter. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(tag = "tags")] pub enum Tags { /// Include all tags All, diff --git a/htsget-config/src/regex_resolver.rs b/htsget-config/src/regex_resolver.rs deleted file mode 100644 index 213d4dd6b..000000000 --- a/htsget-config/src/regex_resolver.rs +++ /dev/null @@ -1,177 +0,0 @@ -use regex::{Error, Regex}; -use serde::{Deserialize, Serialize}; -use tracing::instrument; - -use crate::config::StorageType; -use crate::Format::{Bam, Bcf, Cram, Vcf}; -use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; - -/// Represents an id resolver, which matches the id, replacing the match in the substitution text. -pub trait HtsGetIdResolver { - /// Resolve the id, returning the substituted string if there is a match. - fn resolve_id(&self, query: &Query) -> Option; -} - -/// Determines whether the query matches for use with the resolver. -pub trait QueryMatcher { - /// Does this query match. - fn query_matches(&self, query: &Query) -> bool; -} - -/// A regex resolver is a resolver that matches ids using Regex. -#[derive(Serialize, Debug, Clone, Deserialize)] -#[serde(default)] -pub struct RegexResolver { - #[serde(with = "serde_regex")] - pub regex: Regex, - pub substitution_string: String, - pub server: StorageType, - #[serde(flatten)] - pub match_guard: MatchOnQuery, -} - -/// A query that can be matched with the regex resolver. -#[derive(Serialize, Clone, Debug, Deserialize)] -pub struct MatchOnQuery { - pub format: Vec, - pub class: Vec, - #[serde(with = "serde_regex")] - pub reference_name: Regex, - /// The start and end positions are 0-based. [start, end) - pub start: Interval, - pub end: Interval, - pub fields: Fields, - pub tags: Tags, - pub no_tags: NoTags, -} - -impl Default for MatchOnQuery { - fn default() -> Self { - Self { - format: vec![Bam, Cram, Vcf, Bcf], - class: vec![Class::Body, Class::Header], - reference_name: Regex::new(".*").expect("Expected valid regex expression"), - start: Default::default(), - end: Default::default(), - fields: Fields::All, - tags: Tags::All, - no_tags: NoTags(None), - } - } -} - -impl QueryMatcher for Fields { - fn query_matches(&self, query: &Query) -> bool { - match (self, &query.fields) { - (Fields::All, _) => true, - (Fields::List(self_fields), Fields::List(query_fields)) => self_fields == query_fields, - (Fields::List(_), Fields::All) => false, - } - } -} - -impl QueryMatcher for Tags { - fn query_matches(&self, query: &Query) -> bool { - match (self, &query.tags) { - (Tags::All, _) => true, - (Tags::List(self_tags), Tags::List(query_tags)) => self_tags == query_tags, - (Tags::List(_), Tags::All) => false, - } - } -} - -impl QueryMatcher for NoTags { - fn query_matches(&self, query: &Query) -> bool { - match (self, &query.no_tags) { - (NoTags(None), _) => true, - (NoTags(Some(self_no_tags)), NoTags(Some(query_no_tags))) => self_no_tags == query_no_tags, - (NoTags(Some(_)), NoTags(None)) => false, - } - } -} - -impl QueryMatcher for MatchOnQuery { - fn query_matches(&self, query: &Query) -> bool { - if let Some(reference_name) = &query.reference_name { - self.format.contains(&query.format) - && self.class.contains(&query.class) - && self.reference_name.is_match(reference_name) - && self - .start - .contains(query.interval.start.unwrap_or(u32::MIN)) - && self.end.contains(query.interval.end.unwrap_or(u32::MAX)) - && self.fields.query_matches(query) - && self.fields.query_matches(query) - && self.fields.query_matches(query) - } else { - false - } - } -} - -impl Default for RegexResolver { - fn default() -> Self { - Self::new(".*", "$0", StorageType::default(), MatchOnQuery::default()) - .expect("expected valid resolver") - } -} - -impl From for RegexResolver { - fn from(storage_type: StorageType) -> Self { - Self::new(".*", "$0", storage_type, MatchOnQuery::default()) - .expect("expected valid resolver") - } -} - -impl RegexResolver { - /// Create a new regex resolver. - pub fn new( - regex: &str, - replacement_string: &str, - server: StorageType, - match_guard: MatchOnQuery, - ) -> Result { - Ok(Self { - regex: Regex::new(regex)?, - server, - substitution_string: replacement_string.to_string(), - match_guard, - }) - } -} - -impl HtsGetIdResolver for RegexResolver { - #[instrument(level = "trace", skip(self), ret)] - fn resolve_id(&self, query: &Query) -> Option { - if self.regex.is_match(&query.id) && self.match_guard.query_matches(query) { - Some( - self - .regex - .replace(&query.id, &self.substitution_string) - .to_string(), - ) - } else { - None - } - } -} - -#[cfg(test)] -pub mod tests { - use super::*; - - #[test] - fn resolver_resolve_id() { - let resolver = RegexResolver::new( - ".*", - "$0-test", - StorageType::default(), - MatchOnQuery::default(), - ) - .unwrap(); - assert_eq!( - resolver.resolve_id(&Query::new("id", Bam)).unwrap(), - "id-test" - ); - } -} diff --git a/htsget-config/src/config/aws.rs b/htsget-config/src/regex_resolver/aws.rs similarity index 89% rename from htsget-config/src/config/aws.rs rename to htsget-config/src/regex_resolver/aws.rs index 9a5964453..3e94d47b0 100644 --- a/htsget-config/src/config/aws.rs +++ b/htsget-config/src/regex_resolver/aws.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; /// Configuration for the htsget server. #[derive(Deserialize, Serialize, Debug, Clone, Default)] #[serde(default)] -pub struct AwsS3DataServer { +pub struct S3Resolver { pub bucket: String, pub cors_allow_credentials: bool, pub cors_allow_origin: String, diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs new file mode 100644 index 000000000..94a44732d --- /dev/null +++ b/htsget-config/src/regex_resolver/mod.rs @@ -0,0 +1,224 @@ +use http::uri::Authority; +use regex::{Error, Regex}; +use serde::{Deserialize, Serialize}; +use tracing::instrument; + +use crate::config::{default_localstorage_addr, default_serve_at}; +use crate::Format::{Bam, Bcf, Cram, Vcf}; +use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; +use crate::regex_resolver::aws::S3Resolver; +use crate::regex_resolver::Scheme::Https; + +#[cfg(feature = "s3-storage")] +pub mod aws; + +/// Represents an id resolver, which matches the id, replacing the match in the substitution text. +pub trait HtsGetIdResolver { + /// Resolve the id, returning the substituted string if there is a match. + fn resolve_id(&self, query: &Query) -> Option; +} + +/// Determines whether the query matches for use with the resolver. +pub trait QueryMatcher { + /// Does this query match. + fn query_matches(&self, query: &Query) -> bool; +} + +/// Specify the storage type to use. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type")] +#[non_exhaustive] +pub enum StorageType { + Url(UrlResolver), + #[cfg(feature = "s3-storage")] + S3(S3Resolver), +} + +impl Default for StorageType { + fn default() -> Self { + Self::Url(UrlResolver::default()) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum Scheme { + Http, + Https +} + +impl Default for Scheme { + fn default() -> Self { + Self::Http + } +} + +/// Configuration for the htsget server. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct UrlResolver { + pub scheme: Scheme, + #[serde(with = "http_serde::authority")] + pub authority: Authority, + pub path: String, +} + +impl Default for UrlResolver { + fn default() -> Self { + Self { + scheme: Scheme::default(), + authority: Authority::from_static(default_localstorage_addr()), + path: default_serve_at().to_string() + } + } +} + +/// A regex resolver is a resolver that matches ids using Regex. +#[derive(Serialize, Debug, Clone, Deserialize)] +#[serde(default)] +pub struct RegexResolver { + #[serde(with = "serde_regex")] + pub regex: Regex, + pub substitution_string: String, + pub storage_type: StorageType, + pub guard: QueryGuard, +} + +/// A query that can be matched with the regex resolver. +#[derive(Serialize, Clone, Debug, Deserialize)] +#[serde(default)] +pub struct QueryGuard { + pub match_formats: Vec, + pub match_class: Vec, + #[serde(with = "serde_regex")] + pub match_reference_name: Regex, + /// The start and end positions are 0-based. [start, end) + pub start_interval: Interval, + pub end_interval: Interval, + pub match_fields: Fields, + pub match_tags: Tags, + pub match_no_tags: NoTags, +} + +impl Default for QueryGuard { + fn default() -> Self { + Self { + match_formats: vec![Bam, Cram, Vcf, Bcf], + match_class: vec![Class::Body, Class::Header], + match_reference_name: Regex::new(".*").expect("Expected valid regex expression"), + start_interval: Interval { start: Some(0), end: Some(100) }, + end_interval: Default::default(), + match_fields: Fields::All, + match_tags: Tags::All, + match_no_tags: NoTags(None), + } + } +} + +impl QueryMatcher for Fields { + fn query_matches(&self, query: &Query) -> bool { + match (self, &query.fields) { + (Fields::All, _) => true, + (Fields::List(self_fields), Fields::List(query_fields)) => self_fields == query_fields, + (Fields::List(_), Fields::All) => false, + } + } +} + +impl QueryMatcher for Tags { + fn query_matches(&self, query: &Query) -> bool { + match (self, &query.tags) { + (Tags::All, _) => true, + (Tags::List(self_tags), Tags::List(query_tags)) => self_tags == query_tags, + (Tags::List(_), Tags::All) => false, + } + } +} + +impl QueryMatcher for NoTags { + fn query_matches(&self, query: &Query) -> bool { + match (self, &query.no_tags) { + (NoTags(None), _) => true, + (NoTags(Some(self_no_tags)), NoTags(Some(query_no_tags))) => self_no_tags == query_no_tags, + (NoTags(Some(_)), NoTags(None)) => false, + } + } +} + +impl QueryMatcher for QueryGuard { + fn query_matches(&self, query: &Query) -> bool { + if let Some(reference_name) = &query.reference_name { + self.match_formats.contains(&query.format) + && self.match_class.contains(&query.class) + && self.match_reference_name.is_match(reference_name) + && self + .start_interval + .contains(query.interval.start.unwrap_or(u32::MIN)) + && self.end_interval.contains(query.interval.end.unwrap_or(u32::MAX)) + && self.match_fields.query_matches(query) + && self.match_tags.query_matches(query) + && self.match_no_tags.query_matches(query) + } else { + false + } + } +} + +impl Default for RegexResolver { + fn default() -> Self { + Self::new(StorageType::default(), ".*", "$0", QueryGuard::default()) + .expect("expected valid resolver") + } +} + +impl RegexResolver { + /// Create a new regex resolver. + pub fn new( + storage_type: StorageType, + regex: &str, + replacement_string: &str, + guard: QueryGuard, + ) -> Result { + Ok(Self { + regex: Regex::new(regex)?, + substitution_string: replacement_string.to_string(), + storage_type, + guard, + }) + } +} + +impl HtsGetIdResolver for RegexResolver { + #[instrument(level = "trace", skip(self), ret)] + fn resolve_id(&self, query: &Query) -> Option { + if self.regex.is_match(&query.id) && self.guard.query_matches(query) { + Some( + self + .regex + .replace(&query.id, &self.substitution_string) + .to_string(), + ) + } else { + None + } + } +} + +#[cfg(test)] +pub mod tests { + use super::*; + + #[test] + fn resolver_resolve_id() { + let resolver = RegexResolver::new( + StorageType::default(), + ".*", + "$0-test", + QueryGuard::default(), + ) + .unwrap(); + assert_eq!( + resolver.resolve_id(&Query::new("id", Bam)).unwrap(), + "id-test" + ); + } +} From bb3875ed9f6cb5df3ae8539c5938d885484a6c6a Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 6 Dec 2022 13:19:50 +1100 Subject: [PATCH 12/45] config: add CorsConfig shared struct --- htsget-config/Cargo.toml | 1 + htsget-config/src/config.rs | 42 +++++++++++++++++-------- htsget-config/src/regex_resolver/aws.rs | 2 -- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 779a14d1f..d556bc33d 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -11,6 +11,7 @@ default = ["s3-storage"] [dependencies] noodles = { version = "0.29", features = ["core"] } serde = { version = "1.0", features = ["derive"] } +serde_with = "2.1" serde_regex = "1.1" regex = "1.6" figment = { version = "0.10", features = ["env", "toml"] } diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index 3329df2d8..557f615f7 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -8,6 +8,7 @@ use clap::Parser; use figment::providers::{Env, Format, Serialized, Toml}; use figment::Figment; use serde::{Deserialize, Serialize}; +use serde_with::with_prefix; use tracing::info; use tracing::instrument; use tracing_subscriber::layer::SubscriberExt; @@ -62,14 +63,10 @@ fn default_addr() -> &'static str { "127.0.0.1:8080" } -fn default_ticket_server_origin() -> &'static str { +fn default_server_origin() -> &'static str { "http://localhost:8080" } -fn default_data_server_origin() -> &'static str { - "http://localhost:8081" -} - fn default_path() -> &'static str { "data" } @@ -97,17 +94,38 @@ pub struct Config { pub resolver: Vec, } +with_prefix!(prefix_ticket_server "ticket_server_"); + /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { pub ticket_server_addr: SocketAddr, - pub ticket_server_cors_allow_credentials: bool, - pub ticket_server_cors_allow_origin: String, + #[serde(flatten, with = "prefix_ticket_server")] + pub cors_config: CorsConfig, #[serde(flatten)] pub service_info: ServiceInfo, } +/// Configuration for the htsget server. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct CorsConfig { + pub cors_allow_credentials: bool, + pub cors_allow_origin: String, +} + +impl Default for CorsConfig { + fn default() -> Self { + Self { + cors_allow_credentials: false, + cors_allow_origin: default_server_origin().to_string() + } + } +} + +with_prefix!(prefix_data_server "data_server_"); + /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] @@ -118,8 +136,8 @@ pub struct DataServerConfig { pub data_server_addr: SocketAddr, pub data_server_key: Option, pub data_server_cert: Option, - pub data_server_cors_allow_credentials: bool, - pub data_server_cors_allow_origin: String, + #[serde(flatten, with = "prefix_data_server")] + pub cors_config: CorsConfig, } impl Default for DataServerConfig { @@ -131,8 +149,7 @@ impl Default for DataServerConfig { data_server_addr: default_localstorage_addr().parse().expect("expected valid address"), data_server_key: None, data_server_cert: None, - data_server_cors_allow_credentials: false, - data_server_cors_allow_origin: default_data_server_origin().to_string(), + cors_config: CorsConfig::default(), } } } @@ -157,8 +174,7 @@ impl Default for TicketServerConfig { fn default() -> Self { Self { ticket_server_addr: default_addr().parse().expect("expected valid address"), - ticket_server_cors_allow_credentials: false, - ticket_server_cors_allow_origin: default_ticket_server_origin().to_string(), + cors_config: CorsConfig::default(), service_info: ServiceInfo::default(), } } diff --git a/htsget-config/src/regex_resolver/aws.rs b/htsget-config/src/regex_resolver/aws.rs index 3e94d47b0..fe399aece 100644 --- a/htsget-config/src/regex_resolver/aws.rs +++ b/htsget-config/src/regex_resolver/aws.rs @@ -6,6 +6,4 @@ use serde::{Deserialize, Serialize}; #[serde(default)] pub struct S3Resolver { pub bucket: String, - pub cors_allow_credentials: bool, - pub cors_allow_origin: String, } From 4f3243cf5e3db8796bb88fbc589c21a570ca8fde Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 6 Dec 2022 13:54:07 +1100 Subject: [PATCH 13/45] config: add cors allow header types for cors config --- htsget-config/src/config.rs | 34 ++++++++++++++++++++++++++++++++-- htsget-config/src/lib.rs | 2 +- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index 557f615f7..5d5f1e2c1 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -2,12 +2,16 @@ use std::io; use std::io::ErrorKind; use std::net::SocketAddr; use std::path::PathBuf; +use std::str::FromStr; use crate::regex_resolver::aws::S3Resolver; use clap::Parser; use figment::providers::{Env, Format, Serialized, Toml}; use figment::Figment; -use serde::{Deserialize, Serialize}; +use http::header::HeaderName; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde::de::Error; +use serde::ser::SerializeSeq; use serde_with::with_prefix; use tracing::info; use tracing::instrument; @@ -107,19 +111,45 @@ pub struct TicketServerConfig { pub service_info: ServiceInfo, } +/// Allowed header for cors config. Any allows all headers by sending a wildcard, +/// and mirror allows all headers by mirroring the recieved headers. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged)] +pub enum AllowHeaders { + Any, + Mirror, + #[serde(serialize_with = "serialize_header_names", deserialize_with = "deserialize_header_names")] + List(Vec) +} + +fn serialize_header_names(names: &Vec, serializer: S) -> Result where S: Serializer { + let mut sequence = serializer.serialize_seq(Some(names.len()))?; + for element in names.iter().map(|name| name.as_str()) { + sequence.serialize_element(element)?; + } + sequence.end() +} + +fn deserialize_header_names<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de> { + let names: Vec = Deserialize::deserialize(deserializer)?; + names.into_iter().map(|name| HeaderName::from_str(&name).map_err(|err| Error::custom(err.to_string()))).collect() +} + /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct CorsConfig { pub cors_allow_credentials: bool, pub cors_allow_origin: String, + pub cors_allow_headers: AllowHeaders, } impl Default for CorsConfig { fn default() -> Self { Self { cors_allow_credentials: false, - cors_allow_origin: default_server_origin().to_string() + cors_allow_origin: default_server_origin().to_string(), + cors_allow_headers: AllowHeaders::List(vec![HeaderName::from_static("content-type")]), } } } diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 06ba8c4c0..69003a4e9 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -161,7 +161,7 @@ pub enum Fields { /// Possible values for the tags parameter. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(tag = "tags")] +#[serde(untagged)] pub enum Tags { /// Include all tags All, From 6a2c1779681c652aa5de755c0f10c879c8d36cfd Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 6 Dec 2022 14:10:13 +1100 Subject: [PATCH 14/45] config: add cors max age option --- htsget-config/src/config.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index 5d5f1e2c1..9c8cf910e 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -3,6 +3,7 @@ use std::io::ErrorKind; use std::net::SocketAddr; use std::path::PathBuf; use std::str::FromStr; +use std::time::Duration; use crate::regex_resolver::aws::S3Resolver; use clap::Parser; @@ -58,6 +59,8 @@ The next variables are used to configure the info for the service-info endpoints "#; const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET_"; +/// The maximum amount of time a CORS request can be cached for in seconds. +pub const CORS_MAX_AGE: usize = 86400; pub(crate) fn default_localstorage_addr() -> &'static str { "127.0.0.1:8081" @@ -142,6 +145,7 @@ pub struct CorsConfig { pub cors_allow_credentials: bool, pub cors_allow_origin: String, pub cors_allow_headers: AllowHeaders, + pub cors_max_age: usize } impl Default for CorsConfig { @@ -150,6 +154,7 @@ impl Default for CorsConfig { cors_allow_credentials: false, cors_allow_origin: default_server_origin().to_string(), cors_allow_headers: AllowHeaders::List(vec![HeaderName::from_static("content-type")]), + cors_max_age: CORS_MAX_AGE } } } From 2a8927ddf274db04a7dd353f274aab698084b66a Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 6 Dec 2022 14:34:21 +1100 Subject: [PATCH 15/45] config: add generic allow type configuration option for allow headers and allow methods --- htsget-config/src/config.rs | 38 +++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index 9c8cf910e..66b9ae8b8 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -1,3 +1,4 @@ +use std::fmt::Display; use std::io; use std::io::ErrorKind; use std::net::SocketAddr; @@ -10,6 +11,7 @@ use clap::Parser; use figment::providers::{Env, Format, Serialized, Toml}; use figment::Figment; use http::header::HeaderName; +use http::Method; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde::de::Error; use serde::ser::SerializeSeq; @@ -18,6 +20,7 @@ use tracing::info; use tracing::instrument; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::{EnvFilter, fmt, Registry}; +use crate::config::AllowType::{List, Mirror}; use crate::regex_resolver::RegexResolver; @@ -115,27 +118,36 @@ pub struct TicketServerConfig { } /// Allowed header for cors config. Any allows all headers by sending a wildcard, -/// and mirror allows all headers by mirroring the recieved headers. +/// and mirror allows all headers by mirroring the received headers. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(untagged)] -pub enum AllowHeaders { - Any, +pub enum AllowType { Mirror, - #[serde(serialize_with = "serialize_header_names", deserialize_with = "deserialize_header_names")] - List(Vec) + Any, + #[serde(bound(serialize = "T: AsRef", deserialize = "T: FromStr, T::Err: Display"))] + #[serde(serialize_with = "serialize_allow_types", deserialize_with = "deserialize_allow_types")] + List(Vec) } -fn serialize_header_names(names: &Vec, serializer: S) -> Result where S: Serializer { +fn serialize_allow_types(names: &Vec, serializer: S) -> Result + where + T: AsRef, + S: Serializer +{ let mut sequence = serializer.serialize_seq(Some(names.len()))?; - for element in names.iter().map(|name| name.as_str()) { + for element in names.iter().map(|name| name.as_ref()) { sequence.serialize_element(element)?; } sequence.end() } -fn deserialize_header_names<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de> { +fn deserialize_allow_types<'de, D, T>(deserializer: D) -> Result, D::Error> + where + T: FromStr, + T::Err: Display, + D: Deserializer<'de> +{ let names: Vec = Deserialize::deserialize(deserializer)?; - names.into_iter().map(|name| HeaderName::from_str(&name).map_err(|err| Error::custom(err.to_string()))).collect() + names.into_iter().map(|name| T::from_str(&name).map_err(Error::custom)).collect() } /// Configuration for the htsget server. @@ -144,7 +156,8 @@ fn deserialize_header_names<'de, D>(deserializer: D) -> Result, pub struct CorsConfig { pub cors_allow_credentials: bool, pub cors_allow_origin: String, - pub cors_allow_headers: AllowHeaders, + pub cors_allow_headers: AllowType, + pub cors_allow_methods: AllowType, pub cors_max_age: usize } @@ -153,7 +166,8 @@ impl Default for CorsConfig { Self { cors_allow_credentials: false, cors_allow_origin: default_server_origin().to_string(), - cors_allow_headers: AllowHeaders::List(vec![HeaderName::from_static("content-type")]), + cors_allow_headers: Mirror, + cors_allow_methods: Mirror, cors_max_age: CORS_MAX_AGE } } From 8c17cf816d3f754dfc7bf5b87f7b9af017417704 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 9 Dec 2022 10:16:01 +1100 Subject: [PATCH 16/45] config: add allow origins, and separate out tagged and untagged enum variants --- htsget-config/config.toml | 3 ++- htsget-config/src/config.rs | 54 ++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/htsget-config/config.toml b/htsget-config/config.toml index 3c4262631..f878c3018 100644 --- a/htsget-config/config.toml +++ b/htsget-config/config.toml @@ -6,7 +6,8 @@ data_server_path = "data" data_server_serve_at = "/data" data_server_addr = "127.0.0.1:8081" data_server_cors_allow_credentials = false -data_server_cors_allow_origin = "http://localhost:8081" +data_server_cors_allow_origins = ["http://localhost:8081"] +data_server_cors_allow_methods = ["GET"] [[resolver]] regex = ".*" diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index 66b9ae8b8..804d07811 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -1,4 +1,4 @@ -use std::fmt::Display; +use std::fmt::{Debug, Display, Formatter}; use std::io; use std::io::ErrorKind; use std::net::SocketAddr; @@ -10,17 +10,17 @@ use crate::regex_resolver::aws::S3Resolver; use clap::Parser; use figment::providers::{Env, Format, Serialized, Toml}; use figment::Figment; -use http::header::HeaderName; -use http::Method; +use http::header::{HeaderName, InvalidHeaderValue}; +use http::{HeaderValue as HeaderValueInner, Method}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde::de::Error; use serde::ser::SerializeSeq; -use serde_with::with_prefix; +use serde_with::{DeserializeFromStr, SerializeDisplay, with_prefix}; use tracing::info; use tracing::instrument; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::{EnvFilter, fmt, Registry}; -use crate::config::AllowType::{List, Mirror}; +use crate::config::AllowType::{List}; use crate::regex_resolver::RegexResolver; @@ -117,25 +117,32 @@ pub struct TicketServerConfig { pub service_info: ServiceInfo, } +/// Tagged allow headers for cors config. Either Mirror or Any. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum TaggedAllowTypes { + Mirror, + Any +} + /// Allowed header for cors config. Any allows all headers by sending a wildcard, /// and mirror allows all headers by mirroring the received headers. #[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged)] pub enum AllowType { - Mirror, - Any, - #[serde(bound(serialize = "T: AsRef", deserialize = "T: FromStr, T::Err: Display"))] + TaggedAllowTypes(TaggedAllowTypes), + #[serde(bound(serialize = "T: Display", deserialize = "T: FromStr, T::Err: Display"))] #[serde(serialize_with = "serialize_allow_types", deserialize_with = "deserialize_allow_types")] List(Vec) } fn serialize_allow_types(names: &Vec, serializer: S) -> Result where - T: AsRef, + T: Display, S: Serializer { let mut sequence = serializer.serialize_seq(Some(names.len()))?; - for element in names.iter().map(|name| name.as_ref()) { - sequence.serialize_element(element)?; + for element in names.iter().map(|name| format!("{}", name)) { + sequence.serialize_element(&element)?; } sequence.end() } @@ -150,12 +157,29 @@ fn deserialize_allow_types<'de, D, T>(deserializer: D) -> Result, D::Erro names.into_iter().map(|name| T::from_str(&name).map_err(Error::custom)).collect() } +#[derive(Debug, Clone)] +pub struct HeaderValue(HeaderValueInner); + +impl FromStr for HeaderValue { + type Err = InvalidHeaderValue; + + fn from_str(header: &str) -> Result { + Ok(HeaderValue(HeaderValueInner::from_str(header)?)) + } +} + +impl Display for HeaderValue { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&String::from_utf8_lossy(self.0.as_ref())) + } +} + /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct CorsConfig { pub cors_allow_credentials: bool, - pub cors_allow_origin: String, + pub cors_allow_origins: AllowType, pub cors_allow_headers: AllowType, pub cors_allow_methods: AllowType, pub cors_max_age: usize @@ -165,9 +189,9 @@ impl Default for CorsConfig { fn default() -> Self { Self { cors_allow_credentials: false, - cors_allow_origin: default_server_origin().to_string(), - cors_allow_headers: Mirror, - cors_allow_methods: Mirror, + cors_allow_origins: List(vec![HeaderValue(HeaderValueInner::from_static(default_server_origin()))]), + cors_allow_headers: AllowType::TaggedAllowTypes(TaggedAllowTypes::Mirror), + cors_allow_methods: AllowType::TaggedAllowTypes(TaggedAllowTypes::Mirror), cors_max_age: CORS_MAX_AGE } } From 94fc0e4dddc01a716061c373a8cfab178a45d8ce Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 9 Dec 2022 10:53:24 +1100 Subject: [PATCH 17/45] config: add case insensitive aliases to enum variants --- htsget-config/config.toml | 2 +- htsget-config/src/config.rs | 2 ++ htsget-config/src/regex_resolver/mod.rs | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/htsget-config/config.toml b/htsget-config/config.toml index f878c3018..73a07add3 100644 --- a/htsget-config/config.toml +++ b/htsget-config/config.toml @@ -7,7 +7,7 @@ data_server_serve_at = "/data" data_server_addr = "127.0.0.1:8081" data_server_cors_allow_credentials = false data_server_cors_allow_origins = ["http://localhost:8081"] -data_server_cors_allow_methods = ["GET"] +data_server_cors_allow_methods = "Any" [[resolver]] regex = ".*" diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs index 804d07811..de9ede3a1 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config.rs @@ -120,7 +120,9 @@ pub struct TicketServerConfig { /// Tagged allow headers for cors config. Either Mirror or Any. #[derive(Serialize, Deserialize, Debug, Clone)] pub enum TaggedAllowTypes { + #[serde(alias = "mirror", alias = "MIRROR")] Mirror, + #[serde(alias = "any", alias = "ANY")] Any } diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 94a44732d..d79f9a131 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -29,8 +29,10 @@ pub trait QueryMatcher { #[serde(tag = "type")] #[non_exhaustive] pub enum StorageType { + #[serde(alias = "url", alias = "URL")] Url(UrlResolver), #[cfg(feature = "s3-storage")] + #[serde(alias = "s3")] S3(S3Resolver), } @@ -42,7 +44,9 @@ impl Default for StorageType { #[derive(Serialize, Deserialize, Debug, Clone)] pub enum Scheme { + #[serde(alias = "http", alias = "HTTP")] Http, + #[serde(alias = "https", alias = "HTTPS")] Https } From 3baab72c6bb400f688694b7b0b54e70298b16b4a Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 9 Dec 2022 13:39:40 +1100 Subject: [PATCH 18/45] config: move cors config to separate file --- htsget-config/src/config/cors.rs | 101 +++++++++++++ .../src/{config.rs => config/mod.rs} | 134 ++++-------------- htsget-config/src/lib.rs | 12 +- htsget-config/src/regex_resolver/mod.rs | 7 +- htsget-search/src/lib.rs | 2 +- htsget-search/src/storage/mod.rs | 2 +- 6 files changed, 145 insertions(+), 113 deletions(-) create mode 100644 htsget-config/src/config/cors.rs rename htsget-config/src/{config.rs => config/mod.rs} (72%) diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs new file mode 100644 index 000000000..7db4e30d3 --- /dev/null +++ b/htsget-config/src/config/cors.rs @@ -0,0 +1,101 @@ +use std::fmt::{Display, Formatter}; +use std::str::FromStr; +use http::header::{HeaderName, InvalidHeaderValue, HeaderValue as HeaderValueInner}; +use http::Method; +use serde::{Deserialize, Serialize, Deserializer, Serializer}; +use serde::de::Error; +use serde::ser::SerializeSeq; +use serde_with::with_prefix; +use crate::config::default_server_origin; + +/// The maximum default amount of time a CORS request can be cached for in seconds. +pub const CORS_MAX_AGE: usize = 86400; + +/// Tagged allow headers for cors config. Either Mirror or Any. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum TaggedAllowTypes { + #[serde(alias = "mirror", alias = "MIRROR")] + Mirror, + #[serde(alias = "any", alias = "ANY")] + Any +} + +/// Allowed header for cors config. Any allows all headers by sending a wildcard, +/// and mirror allows all headers by mirroring the received headers. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged)] +pub enum AllowType { + TaggedAllowTypes(TaggedAllowTypes), + #[serde(bound(serialize = "T: Display", deserialize = "T: FromStr, T::Err: Display"))] + #[serde(serialize_with = "serialize_allow_types", deserialize_with = "deserialize_allow_types")] + List(Vec) +} + +fn serialize_allow_types(names: &Vec, serializer: S) -> Result + where + T: Display, + S: Serializer +{ + let mut sequence = serializer.serialize_seq(Some(names.len()))?; + for element in names.iter().map(|name| format!("{}", name)) { + sequence.serialize_element(&element)?; + } + sequence.end() +} + +fn deserialize_allow_types<'de, D, T>(deserializer: D) -> Result, D::Error> + where + T: FromStr, + T::Err: Display, + D: Deserializer<'de> +{ + let names: Vec = Deserialize::deserialize(deserializer)?; + names.into_iter().map(|name| T::from_str(&name).map_err(Error::custom)).collect() +} + +#[derive(Debug, Clone)] +pub struct HeaderValue(HeaderValueInner); + +impl FromStr for HeaderValue { + type Err = InvalidHeaderValue; + + fn from_str(header: &str) -> Result { + Ok(HeaderValue(HeaderValueInner::from_str(header)?)) + } +} + +impl Display for HeaderValue { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&String::from_utf8_lossy(self.0.as_ref())) + } +} + +with_prefix!(prefix_cors "cors_"); + +/// Configuration for the htsget server. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct CorsConfig { + #[serde(with = "prefix_cors")] + pub allow_credentials: bool, + #[serde(with = "prefix_cors")] + pub allow_origins: AllowType, + #[serde(with = "prefix_cors")] + pub allow_headers: AllowType, + #[serde(with = "prefix_cors")] + pub allow_methods: AllowType, + #[serde(with = "prefix_cors")] + pub max_age: usize +} + +impl Default for CorsConfig { + fn default() -> Self { + Self { + allow_credentials: false, + allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static(default_server_origin()))]), + allow_headers: AllowType::TaggedAllowTypes(TaggedAllowTypes::Mirror), + allow_methods: AllowType::TaggedAllowTypes(TaggedAllowTypes::Mirror), + max_age: CORS_MAX_AGE + } + } +} \ No newline at end of file diff --git a/htsget-config/src/config.rs b/htsget-config/src/config/mod.rs similarity index 72% rename from htsget-config/src/config.rs rename to htsget-config/src/config/mod.rs index de9ede3a1..8eb23c33c 100644 --- a/htsget-config/src/config.rs +++ b/htsget-config/src/config/mod.rs @@ -1,3 +1,5 @@ +pub mod cors; + use std::fmt::{Debug, Display, Formatter}; use std::io; use std::io::ErrorKind; @@ -20,7 +22,7 @@ use tracing::info; use tracing::instrument; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::{EnvFilter, fmt, Registry}; -use crate::config::AllowType::{List}; +use crate::config::cors::CorsConfig; use crate::regex_resolver::RegexResolver; @@ -62,8 +64,6 @@ The next variables are used to configure the info for the service-info endpoints "#; const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET_"; -/// The maximum amount of time a CORS request can be cached for in seconds. -pub const CORS_MAX_AGE: usize = 86400; pub(crate) fn default_localstorage_addr() -> &'static str { "127.0.0.1:8081" @@ -101,7 +101,7 @@ pub struct Config { pub ticket_server_config: TicketServerConfig, #[serde(flatten)] pub data_server_config: DataServerConfig, - pub resolver: Vec, + pub resolvers: Vec, } with_prefix!(prefix_ticket_server "ticket_server_"); @@ -110,121 +110,43 @@ with_prefix!(prefix_ticket_server "ticket_server_"); #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { - pub ticket_server_addr: SocketAddr, + #[serde(with = "prefix_ticket_server")] + pub addr: SocketAddr, #[serde(flatten, with = "prefix_ticket_server")] - pub cors_config: CorsConfig, + pub cors: CorsConfig, #[serde(flatten)] pub service_info: ServiceInfo, } -/// Tagged allow headers for cors config. Either Mirror or Any. -#[derive(Serialize, Deserialize, Debug, Clone)] -pub enum TaggedAllowTypes { - #[serde(alias = "mirror", alias = "MIRROR")] - Mirror, - #[serde(alias = "any", alias = "ANY")] - Any -} - -/// Allowed header for cors config. Any allows all headers by sending a wildcard, -/// and mirror allows all headers by mirroring the received headers. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(untagged)] -pub enum AllowType { - TaggedAllowTypes(TaggedAllowTypes), - #[serde(bound(serialize = "T: Display", deserialize = "T: FromStr, T::Err: Display"))] - #[serde(serialize_with = "serialize_allow_types", deserialize_with = "deserialize_allow_types")] - List(Vec) -} - -fn serialize_allow_types(names: &Vec, serializer: S) -> Result - where - T: Display, - S: Serializer -{ - let mut sequence = serializer.serialize_seq(Some(names.len()))?; - for element in names.iter().map(|name| format!("{}", name)) { - sequence.serialize_element(&element)?; - } - sequence.end() -} - -fn deserialize_allow_types<'de, D, T>(deserializer: D) -> Result, D::Error> - where - T: FromStr, - T::Err: Display, - D: Deserializer<'de> -{ - let names: Vec = Deserialize::deserialize(deserializer)?; - names.into_iter().map(|name| T::from_str(&name).map_err(Error::custom)).collect() -} - -#[derive(Debug, Clone)] -pub struct HeaderValue(HeaderValueInner); - -impl FromStr for HeaderValue { - type Err = InvalidHeaderValue; - - fn from_str(header: &str) -> Result { - Ok(HeaderValue(HeaderValueInner::from_str(header)?)) - } -} - -impl Display for HeaderValue { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&String::from_utf8_lossy(self.0.as_ref())) - } -} - -/// Configuration for the htsget server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct CorsConfig { - pub cors_allow_credentials: bool, - pub cors_allow_origins: AllowType, - pub cors_allow_headers: AllowType, - pub cors_allow_methods: AllowType, - pub cors_max_age: usize -} - -impl Default for CorsConfig { - fn default() -> Self { - Self { - cors_allow_credentials: false, - cors_allow_origins: List(vec![HeaderValue(HeaderValueInner::from_static(default_server_origin()))]), - cors_allow_headers: AllowType::TaggedAllowTypes(TaggedAllowTypes::Mirror), - cors_allow_methods: AllowType::TaggedAllowTypes(TaggedAllowTypes::Mirror), - cors_max_age: CORS_MAX_AGE - } - } -} - with_prefix!(prefix_data_server "data_server_"); /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct DataServerConfig { - pub start_data_server: bool, - pub data_server_path: PathBuf, - pub data_server_serve_at: PathBuf, - pub data_server_addr: SocketAddr, - pub data_server_key: Option, - pub data_server_cert: Option, + #[serde(with = "prefix_data_server")] + pub path: PathBuf, + #[serde(with = "prefix_data_server")] + pub serve_at: PathBuf, + #[serde(with = "prefix_data_server")] + pub addr: SocketAddr, + #[serde(with = "prefix_data_server")] + pub key: Option, + #[serde(with = "prefix_data_server")] + pub cert: Option, #[serde(flatten, with = "prefix_data_server")] - pub cors_config: CorsConfig, + pub cors: CorsConfig, } impl Default for DataServerConfig { fn default() -> Self { Self { - start_data_server: true, - data_server_path: default_path().into(), - data_server_serve_at: default_serve_at().into(), - data_server_addr: default_localstorage_addr().parse().expect("expected valid address"), - data_server_key: None, - data_server_cert: None, - cors_config: CorsConfig::default(), + path: default_path().into(), + serve_at: default_serve_at().into(), + addr: default_localstorage_addr().parse().expect("expected valid address"), + key: None, + cert: None, + cors: CorsConfig::default(), } } } @@ -248,8 +170,8 @@ pub struct ServiceInfo { impl Default for TicketServerConfig { fn default() -> Self { Self { - ticket_server_addr: default_addr().parse().expect("expected valid address"), - cors_config: CorsConfig::default(), + addr: default_addr().parse().expect("expected valid address"), + cors: CorsConfig::default(), service_info: ServiceInfo::default(), } } @@ -260,7 +182,7 @@ impl Default for Config { Self { ticket_server_config: Default::default(), data_server_config: Default::default(), - resolver: vec![RegexResolver::default(), RegexResolver::default()], + resolvers: vec![RegexResolver::default(), RegexResolver::default()], } } } @@ -276,7 +198,7 @@ impl Config { pub fn from_env(config: PathBuf) -> io::Result { let config = Figment::from(Serialized::defaults(Config::default())) .merge(Toml::file(config)) - .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX)) + .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX).split("_")) .extract() .map_err(|err| { io::Error::new(ErrorKind::Other, format!("failed to parse config: {}", err)) diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 69003a4e9..85065678a 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -13,11 +13,15 @@ pub mod regex_resolver; /// An enumeration with all the possible formats. #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "UPPERCASE")] +#[serde(rename_all(serialize = "UPPERCASE"))] pub enum Format { + #[serde(alias = "bam", alias = "BAM")] Bam, + #[serde(alias = "cram", alias = "CRAM")] Cram, + #[serde(alias = "vcf", alias = "VCF")] Vcf, + #[serde(alias = "bcf", alias = "BCF")] Bcf, } @@ -71,9 +75,11 @@ impl fmt::Display for Format { } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase"))] pub enum Class { + #[serde(alias = "header", alias = "HEADER")] Header, + #[serde(alias = "body", alias = "BODY")] Body, } @@ -154,6 +160,7 @@ impl Interval { #[serde(untagged)] pub enum Fields { /// Include all fields + #[serde(alias = "all", alias = "ALL")] All, /// List of fields to include List(Vec), @@ -164,6 +171,7 @@ pub enum Fields { #[serde(untagged)] pub enum Tags { /// Include all tags + #[serde(alias = "all", alias = "ALL")] All, /// List of tags to include List(Vec), diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index d79f9a131..a7c63e58d 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -1,6 +1,7 @@ use http::uri::Authority; use regex::{Error, Regex}; use serde::{Deserialize, Serialize}; +use serde_with::with_prefix; use tracing::instrument; use crate::config::{default_localstorage_addr, default_serve_at}; @@ -13,7 +14,7 @@ use crate::regex_resolver::Scheme::Https; pub mod aws; /// Represents an id resolver, which matches the id, replacing the match in the substitution text. -pub trait HtsGetIdResolver { +pub trait IdResolver { /// Resolve the id, returning the substituted string if there is a match. fn resolve_id(&self, query: &Query) -> Option; } @@ -83,8 +84,8 @@ pub struct RegexResolver { #[serde(with = "serde_regex")] pub regex: Regex, pub substitution_string: String, - pub storage_type: StorageType, pub guard: QueryGuard, + pub storage_type: StorageType, } /// A query that can be matched with the regex resolver. @@ -191,7 +192,7 @@ impl RegexResolver { } } -impl HtsGetIdResolver for RegexResolver { +impl IdResolver for RegexResolver { #[instrument(level = "trace", skip(self), ret)] fn resolve_id(&self, query: &Query) -> Option { if self.regex.is_match(&query.id) && self.guard.query_matches(query) { diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index a348c4983..0e7a5a67c 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -3,7 +3,7 @@ pub use htsget_config::config::aws::AwsS3DataServer; pub use htsget_config::config::{ Config, LocalDataServer, ServiceInfo, StorageType, TicketServerConfig, }; -pub use htsget_config::regex_resolver::{HtsGetIdResolver, RegexResolver}; +pub use htsget_config::regex_resolver::{IdResolver, RegexResolver}; pub mod htsget; pub mod storage; diff --git a/htsget-search/src/storage/mod.rs b/htsget-search/src/storage/mod.rs index b7f74f806..b320e837d 100644 --- a/htsget-search/src/storage/mod.rs +++ b/htsget-search/src/storage/mod.rs @@ -19,7 +19,7 @@ use tracing::instrument; use crate::htsget::{Headers, Url}; use crate::storage::data_server::CORS_MAX_AGE; use crate::storage::StorageError::DataServerError; -use crate::{HtsGetIdResolver, RegexResolver}; +use crate::{IdResolver, RegexResolver}; #[cfg(feature = "s3-storage")] pub mod aws; From b450848fed401c7590d94a07574fb6eeae068fef Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 9 Dec 2022 14:55:31 +1100 Subject: [PATCH 19/45] config: add expose headers cors option --- htsget-config/src/config/cors.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs index 7db4e30d3..2df7eb475 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/cors.rs @@ -20,12 +20,21 @@ pub enum TaggedAllowTypes { Any } +/// Tagged allow headers for cors config. Either Mirror or Any. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum TaggedAnyAllowType { + #[serde(alias = "mirror", alias = "MIRROR")] + Mirror, + #[serde(alias = "any", alias = "ANY")] + Any +} + /// Allowed header for cors config. Any allows all headers by sending a wildcard, /// and mirror allows all headers by mirroring the received headers. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] -pub enum AllowType { - TaggedAllowTypes(TaggedAllowTypes), +pub enum AllowType { + Tagged(Tagged), #[serde(bound(serialize = "T: Display", deserialize = "T: FromStr, T::Err: Display"))] #[serde(serialize_with = "serialize_allow_types", deserialize_with = "deserialize_allow_types")] List(Vec) @@ -85,7 +94,9 @@ pub struct CorsConfig { #[serde(with = "prefix_cors")] pub allow_methods: AllowType, #[serde(with = "prefix_cors")] - pub max_age: usize + pub max_age: usize, + #[serde(with = "prefix_cors")] + pub expose_headers: AllowType, } impl Default for CorsConfig { @@ -93,9 +104,10 @@ impl Default for CorsConfig { Self { allow_credentials: false, allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static(default_server_origin()))]), - allow_headers: AllowType::TaggedAllowTypes(TaggedAllowTypes::Mirror), - allow_methods: AllowType::TaggedAllowTypes(TaggedAllowTypes::Mirror), - max_age: CORS_MAX_AGE + allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), + allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), + max_age: CORS_MAX_AGE, + expose_headers: AllowType::Tagged(TaggedAnyAllowType::Any), } } } \ No newline at end of file From 5c7ad2c65515637eee1102bb5dc0479797c39344 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 12 Dec 2022 07:51:00 +1100 Subject: [PATCH 20/45] config: allow configuring multiple data servers --- htsget-config/config.toml | 54 +++++++++++++++++--------------- htsget-config/src/config/cors.rs | 35 +++++++-------------- htsget-config/src/config/mod.rs | 48 +++++++++++----------------- 3 files changed, 58 insertions(+), 79 deletions(-) diff --git a/htsget-config/config.toml b/htsget-config/config.toml index 73a07add3..683c9b486 100644 --- a/htsget-config/config.toml +++ b/htsget-config/config.toml @@ -1,26 +1,28 @@ -ticket_server_addr = "127.0.0.1:8080" -ticket_server_cors_allow_credentials = false -ticket_server_cors_allow_origin = "http://localhost:8080" -start_data_server = true -data_server_path = "data" -data_server_serve_at = "/data" -data_server_addr = "127.0.0.1:8081" -data_server_cors_allow_credentials = false -data_server_cors_allow_origins = ["http://localhost:8081"] -data_server_cors_allow_methods = "Any" - -[[resolver]] -regex = ".*" -substitution_string = "$0" - -storage_type.type = "Url" -storage_type.scheme = "Https" -storage_type.authority = "127.0.0.1:8081" -storage_type.path = "/data" - -[resolver.guard] -match_formats = ["BAM"] -start_interval.start = 100 -match_fields = ["field1"] -match_no_tags = ["tag1"] - +ticket_server_addr = "127.0.0.1:8082" +#ticket_server_cors_allow_credentials = false +#ticket_server_cors_allow_origin = "http://localhost:8080" +#start_data_server = true +#data_server_path = "data" +#data_server_serve_at = "/data" +#data_server_config = "None" +data_server_config = [] +#data_server_addr = "127.0.0.1:8082" +#data_server_cors_allow_credentials = false +#data_server_cors_allow_origins = ["http://localhost:8081"] +#data_server_cors_allow_methods = "Any" +# +#[[resolver]] +#regex = ".*" +#substitution_string = "$0" +# +#storage_type.type = "Url" +#storage_type.scheme = "Https" +#storage_type.authority = "127.0.0.1:8081" +#storage_type.path = "/data" +# +#[resolver.guard] +#match_formats = ["BAM"] +#start_interval.start = 100 +#match_fields = ["field1"] +#match_no_tags = ["tag1"] +# diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs index 2df7eb475..5959d54d1 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/cors.rs @@ -5,7 +5,6 @@ use http::Method; use serde::{Deserialize, Serialize, Deserializer, Serializer}; use serde::de::Error; use serde::ser::SerializeSeq; -use serde_with::with_prefix; use crate::config::default_server_origin; /// The maximum default amount of time a CORS request can be cached for in seconds. @@ -23,8 +22,6 @@ pub enum TaggedAllowTypes { /// Tagged allow headers for cors config. Either Mirror or Any. #[derive(Serialize, Deserialize, Debug, Clone)] pub enum TaggedAnyAllowType { - #[serde(alias = "mirror", alias = "MIRROR")] - Mirror, #[serde(alias = "any", alias = "ANY")] Any } @@ -79,35 +76,27 @@ impl Display for HeaderValue { } } -with_prefix!(prefix_cors "cors_"); - /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct CorsConfig { - #[serde(with = "prefix_cors")] - pub allow_credentials: bool, - #[serde(with = "prefix_cors")] - pub allow_origins: AllowType, - #[serde(with = "prefix_cors")] - pub allow_headers: AllowType, - #[serde(with = "prefix_cors")] - pub allow_methods: AllowType, - #[serde(with = "prefix_cors")] - pub max_age: usize, - #[serde(with = "prefix_cors")] - pub expose_headers: AllowType, + pub cors_allow_credentials: bool, + pub cors_allow_origins: AllowType, + pub cors_allow_headers: AllowType, + pub cors_allow_methods: AllowType, + pub cors_max_age: usize, + pub cors_expose_headers: AllowType, } impl Default for CorsConfig { fn default() -> Self { Self { - allow_credentials: false, - allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static(default_server_origin()))]), - allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), - allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), - max_age: CORS_MAX_AGE, - expose_headers: AllowType::Tagged(TaggedAnyAllowType::Any), + cors_allow_credentials: false, + cors_allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static(default_server_origin()))]), + cors_allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), + cors_allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), + cors_max_age: CORS_MAX_AGE, + cors_expose_headers: AllowType::List(vec![]), } } } \ No newline at end of file diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 8eb23c33c..d540783d9 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -15,9 +15,8 @@ use figment::Figment; use http::header::{HeaderName, InvalidHeaderValue}; use http::{HeaderValue as HeaderValueInner, Method}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use serde::de::Error; -use serde::ser::SerializeSeq; -use serde_with::{DeserializeFromStr, SerializeDisplay, with_prefix}; +use serde::de::Error as DeError; +use serde::ser::Error as SeError; use tracing::info; use tracing::instrument; use tracing_subscriber::layer::SubscriberExt; @@ -98,52 +97,41 @@ pub struct Args { #[serde(default)] pub struct Config { #[serde(flatten)] - pub ticket_server_config: TicketServerConfig, - #[serde(flatten)] - pub data_server_config: DataServerConfig, + pub ticket_server: TicketServerConfig, + pub data_server: Vec, pub resolvers: Vec, } -with_prefix!(prefix_ticket_server "ticket_server_"); - /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { - #[serde(with = "prefix_ticket_server")] - pub addr: SocketAddr, - #[serde(flatten, with = "prefix_ticket_server")] + pub ticket_server_addr: SocketAddr, + #[serde(flatten)] pub cors: CorsConfig, #[serde(flatten)] pub service_info: ServiceInfo, } -with_prefix!(prefix_data_server "data_server_"); - /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct DataServerConfig { - #[serde(with = "prefix_data_server")] + pub addr: SocketAddr, pub path: PathBuf, - #[serde(with = "prefix_data_server")] pub serve_at: PathBuf, - #[serde(with = "prefix_data_server")] - pub addr: SocketAddr, - #[serde(with = "prefix_data_server")] pub key: Option, - #[serde(with = "prefix_data_server")] pub cert: Option, - #[serde(flatten, with = "prefix_data_server")] + #[serde(flatten)] pub cors: CorsConfig, } impl Default for DataServerConfig { fn default() -> Self { Self { + addr: default_localstorage_addr().parse().expect("expected valid address"), path: default_path().into(), serve_at: default_serve_at().into(), - addr: default_localstorage_addr().parse().expect("expected valid address"), key: None, cert: None, cors: CorsConfig::default(), @@ -170,7 +158,7 @@ pub struct ServiceInfo { impl Default for TicketServerConfig { fn default() -> Self { Self { - addr: default_addr().parse().expect("expected valid address"), + ticket_server_addr: default_addr().parse().expect("expected valid address"), cors: CorsConfig::default(), service_info: ServiceInfo::default(), } @@ -180,8 +168,8 @@ impl Default for TicketServerConfig { impl Default for Config { fn default() -> Self { Self { - ticket_server_config: Default::default(), - data_server_config: Default::default(), + ticket_server: TicketServerConfig::default(), + data_server: vec![DataServerConfig::default()], resolvers: vec![RegexResolver::default(), RegexResolver::default()], } } @@ -290,12 +278,12 @@ mod tests { // assert_eq!(config.resolver.substitution_string, "$0-test"); // } - #[test] - fn config_service_info_id() { - std::env::set_var("HTSGET_ID", "id"); - let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - assert_eq!(config.ticket_server_config.service_info.id.unwrap(), "id"); - } + // #[test] + // fn config_service_info_id() { + // std::env::set_var("HTSGET_ID", "id"); + // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); + // assert_eq!(config.ticket_server_config.service_info.id.unwrap(), "id"); + // } // #[cfg(feature = "s3-storage")] // #[test] From bc50c7de8ebc05d2fd7a6f18a482040d5ae64bbe Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 12 Dec 2022 13:52:15 +1100 Subject: [PATCH 21/45] config: remove public fields, add public getters --- htsget-config/src/config/cors.rs | 40 ++++- htsget-config/src/config/mod.rs | 230 +++++++++++++++++++++--- htsget-config/src/lib.rs | 64 +++++-- htsget-config/src/regex_resolver/aws.rs | 8 +- htsget-config/src/regex_resolver/mod.rs | 127 +++++++++++-- 5 files changed, 410 insertions(+), 59 deletions(-) diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs index 5959d54d1..a70ff61fb 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/cors.rs @@ -8,7 +8,7 @@ use serde::ser::SerializeSeq; use crate::config::default_server_origin; /// The maximum default amount of time a CORS request can be cached for in seconds. -pub const CORS_MAX_AGE: usize = 86400; +const CORS_MAX_AGE: usize = 86400; /// Tagged allow headers for cors config. Either Mirror or Any. #[derive(Serialize, Deserialize, Debug, Clone)] @@ -80,12 +80,38 @@ impl Display for HeaderValue { #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct CorsConfig { - pub cors_allow_credentials: bool, - pub cors_allow_origins: AllowType, - pub cors_allow_headers: AllowType, - pub cors_allow_methods: AllowType, - pub cors_max_age: usize, - pub cors_expose_headers: AllowType, + cors_allow_credentials: bool, + cors_allow_origins: AllowType, + cors_allow_headers: AllowType, + cors_allow_methods: AllowType, + cors_max_age: usize, + cors_expose_headers: AllowType, +} + +impl CorsConfig { + pub fn allow_credentials(&self) -> bool { + self.cors_allow_credentials + } + + pub fn allow_origins(&self) -> &AllowType { + &self.cors_allow_origins + } + + pub fn allow_headers(&self) -> &AllowType { + &self.cors_allow_headers + } + + pub fn allow_methods(&self) -> &AllowType { + &self.cors_allow_methods + } + + pub fn max_age(&self) -> usize { + self.cors_max_age + } + + pub fn expose_headers(&self) -> &AllowType { + &self.cors_expose_headers + } } impl Default for CorsConfig { diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index d540783d9..52ba10759 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -21,7 +21,7 @@ use tracing::info; use tracing::instrument; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::{EnvFilter, fmt, Registry}; -use crate::config::cors::CorsConfig; +use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAnyAllowType}; use crate::regex_resolver::RegexResolver; @@ -87,7 +87,7 @@ pub(crate) fn default_serve_at() -> &'static str { /// The command line arguments allowed for the htsget-rs executables. #[derive(Parser, Debug)] #[command(author, version, about, long_about = USAGE)] -pub struct Args { +struct Args { #[arg(short, long, env = "HTSGET_CONFIG")] config: PathBuf, } @@ -97,33 +97,161 @@ pub struct Args { #[serde(default)] pub struct Config { #[serde(flatten)] - pub ticket_server: TicketServerConfig, - pub data_server: Vec, - pub resolvers: Vec, + ticket_server: TicketServerConfig, + data_server: Vec, + resolvers: Vec, } /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { - pub ticket_server_addr: SocketAddr, + ticket_server_addr: SocketAddr, #[serde(flatten)] - pub cors: CorsConfig, + cors: CorsConfig, #[serde(flatten)] - pub service_info: ServiceInfo, + service_info: ServiceInfo, +} + +impl TicketServerConfig { + pub fn addr(&self) -> SocketAddr { + self.ticket_server_addr + } + + pub fn cors(&self) -> &CorsConfig { + &self.cors + } + + pub fn service_info(&self) -> &ServiceInfo { + &self.service_info + } + + pub fn allow_credentials(&self) -> bool { + self.cors.allow_credentials() + } + + pub fn allow_origins(&self) -> &AllowType { + self.cors.allow_origins() + } + + pub fn allow_headers(&self) -> &AllowType { + self.cors.allow_headers() + } + + pub fn allow_methods(&self) -> &AllowType { + self.cors.allow_methods() + } + + pub fn max_age(&self) -> usize { + self.cors.max_age() + } + + pub fn expose_headers(&self) -> &AllowType { + self.cors.expose_headers() + } + + pub fn id(&self) -> &Option { + self.service_info.id() + } + + pub fn name(&self) -> &Option { + self.service_info.name() + } + + pub fn version(&self) -> &Option { + self.service_info.version() + } + + pub fn organization_name(&self) -> &Option { + self.service_info.organization_name() + } + + pub fn organization_url(&self) -> &Option { + self.service_info.organization_url() + } + + pub fn contact_url(&self) -> &Option { + self.service_info.contact_url() + } + + pub fn documentation_url(&self) -> &Option { + self.service_info.documentation_url() + } + + pub fn created_at(&self) -> &Option { + self.service_info.created_at() + } + + pub fn updated_at(&self) -> &Option { + self.service_info.updated_at() + } + + pub fn environment(&self) -> &Option { + self.service_info.environment() + } } /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct DataServerConfig { - pub addr: SocketAddr, - pub path: PathBuf, - pub serve_at: PathBuf, - pub key: Option, - pub cert: Option, + addr: SocketAddr, + path: PathBuf, + serve_at: PathBuf, + key: Option, + cert: Option, #[serde(flatten)] - pub cors: CorsConfig, + cors: CorsConfig, +} + +impl DataServerConfig { + pub fn addr(&self) -> SocketAddr { + self.addr + } + + pub fn path(&self) -> &PathBuf { + &self.path + } + + pub fn serve_at(&self) -> &PathBuf { + &self.serve_at + } + + pub fn key(&self) -> &Option { + &self.key + } + + pub fn cert(&self) -> &Option { + &self.cert + } + + pub fn cors(&self) -> &CorsConfig { + &self.cors + } + + pub fn allow_credentials(&self) -> bool { + self.cors.allow_credentials() + } + + pub fn allow_origins(&self) -> &AllowType { + self.cors.allow_origins() + } + + pub fn allow_headers(&self) -> &AllowType { + self.cors.allow_headers() + } + + pub fn allow_methods(&self) -> &AllowType { + self.cors.allow_methods() + } + + pub fn max_age(&self) -> usize { + self.cors.max_age() + } + + pub fn expose_headers(&self) -> &AllowType { + self.cors.expose_headers() + } } impl Default for DataServerConfig { @@ -143,16 +271,58 @@ impl Default for DataServerConfig { #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct ServiceInfo { - pub id: Option, - pub name: Option, - pub version: Option, - pub organization_name: Option, - pub organization_url: Option, - pub contact_url: Option, - pub documentation_url: Option, - pub created_at: Option, - pub updated_at: Option, - pub environment: Option, + id: Option, + name: Option, + version: Option, + organization_name: Option, + organization_url: Option, + contact_url: Option, + documentation_url: Option, + created_at: Option, + updated_at: Option, + environment: Option, +} + +impl ServiceInfo { + pub fn id(&self) -> &Option { + &self.id + } + + pub fn name(&self) -> &Option { + &self.name + } + + pub fn version(&self) -> &Option { + &self.version + } + + pub fn organization_name(&self) -> &Option { + &self.organization_name + } + + pub fn organization_url(&self) -> &Option { + &self.organization_url + } + + pub fn contact_url(&self) -> &Option { + &self.contact_url + } + + pub fn documentation_url(&self) -> &Option { + &self.documentation_url + } + + pub fn created_at(&self) -> &Option { + &self.created_at + } + + pub fn updated_at(&self) -> &Option { + &self.updated_at + } + + pub fn environment(&self) -> &Option { + &self.environment + } } impl Default for TicketServerConfig { @@ -212,6 +382,18 @@ impl Config { Ok(()) } + + pub fn ticket_server(&self) -> &TicketServerConfig { + &self.ticket_server + } + + pub fn data_server(&self) -> &Vec { + &self.data_server + } + + pub fn resolvers(&self) -> &Vec { + &self.resolvers + } } #[cfg(test)] diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 85065678a..66a7cc3f7 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -74,7 +74,7 @@ impl fmt::Display for Format { } } -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[derive(Copy, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] #[serde(rename_all(serialize = "lowercase"))] pub enum Class { #[serde(alias = "header", alias = "HEADER")] @@ -85,10 +85,10 @@ pub enum Class { /// An interval represents the start (0-based, inclusive) and end (0-based exclusive) ranges of the /// query. -#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] pub struct Interval { - pub start: Option, - pub end: Option, + start: Option, + end: Option, } impl Interval { @@ -153,6 +153,14 @@ impl Interval { ) }) } + + pub fn start(&self) -> Option { + self.start + } + + pub fn end(&self) -> Option { + self.end + } } /// Possible values for the fields parameter. @@ -185,16 +193,16 @@ pub struct NoTags(pub Option>); /// a search for either of `reads` or `variants`. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Query { - pub id: String, - pub format: Format, - pub class: Class, + id: String, + format: Format, + class: Class, /// Reference name - pub reference_name: Option, + reference_name: Option, /// The start and end positions are 0-based. [start, end) - pub interval: Interval, - pub fields: Fields, - pub tags: Tags, - pub no_tags: NoTags, + interval: Interval, + fields: Fields, + tags: Tags, + no_tags: NoTags, } impl Query { @@ -252,6 +260,38 @@ impl Query { )); self } + + pub fn id(&self) -> &str { + &self.id + } + + pub fn format(&self) -> Format { + self.format + } + + pub fn class(&self) -> Class { + self.class + } + + pub fn reference_name(&self) -> &Option { + &self.reference_name + } + + pub fn interval(&self) -> Interval { + self.interval + } + + pub fn fields(&self) -> &Fields { + &self.fields + } + + pub fn tags(&self) -> &Tags { + &self.tags + } + + pub fn no_tags(&self) -> &NoTags { + &self.no_tags + } } #[cfg(test)] diff --git a/htsget-config/src/regex_resolver/aws.rs b/htsget-config/src/regex_resolver/aws.rs index fe399aece..bb25fd826 100644 --- a/htsget-config/src/regex_resolver/aws.rs +++ b/htsget-config/src/regex_resolver/aws.rs @@ -5,5 +5,11 @@ use serde::{Deserialize, Serialize}; #[derive(Deserialize, Serialize, Debug, Clone, Default)] #[serde(default)] pub struct S3Resolver { - pub bucket: String, + bucket: String, +} + +impl S3Resolver { + pub fn bucket(&self) -> &str { + &self.bucket + } } diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index a7c63e58d..6b6d2b013 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -61,10 +61,24 @@ impl Default for Scheme { #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct UrlResolver { - pub scheme: Scheme, + scheme: Scheme, #[serde(with = "http_serde::authority")] - pub authority: Authority, - pub path: String, + authority: Authority, + path: String, +} + +impl UrlResolver { + pub fn scheme(&self) -> &Scheme { + &self.scheme + } + + pub fn authority(&self) -> &Authority { + &self.authority + } + + pub fn path(&self) -> &str { + &self.path + } } impl Default for UrlResolver { @@ -82,26 +96,61 @@ impl Default for UrlResolver { #[serde(default)] pub struct RegexResolver { #[serde(with = "serde_regex")] - pub regex: Regex, - pub substitution_string: String, - pub guard: QueryGuard, - pub storage_type: StorageType, + regex: Regex, + // todo: should match guard be allowed as variables inside the substitution string? + substitution_string: String, + guard: QueryGuard, + storage_type: StorageType, } /// A query that can be matched with the regex resolver. #[derive(Serialize, Clone, Debug, Deserialize)] #[serde(default)] pub struct QueryGuard { - pub match_formats: Vec, - pub match_class: Vec, + match_formats: Vec, + match_class: Vec, #[serde(with = "serde_regex")] - pub match_reference_name: Regex, + match_reference_name: Regex, /// The start and end positions are 0-based. [start, end) - pub start_interval: Interval, - pub end_interval: Interval, - pub match_fields: Fields, - pub match_tags: Tags, - pub match_no_tags: NoTags, + start_interval: Interval, + end_interval: Interval, + match_fields: Fields, + match_tags: Tags, + match_no_tags: NoTags, +} + +impl QueryGuard { + pub fn match_formats(&self) -> &Vec { + &self.match_formats + } + + pub fn match_class(&self) -> &Vec { + &self.match_class + } + + pub fn match_reference_name(&self) -> &Regex { + &self.match_reference_name + } + + pub fn start_interval(&self) -> Interval { + self.start_interval + } + + pub fn end_interval(&self) -> Interval { + self.end_interval + } + + pub fn match_fields(&self) -> &Fields { + &self.match_fields + } + + pub fn match_tags(&self) -> &Tags { + &self.match_tags + } + + pub fn match_no_tags(&self) -> &NoTags { + &self.match_no_tags + } } impl Default for QueryGuard { @@ -190,6 +239,54 @@ impl RegexResolver { guard, }) } + + pub fn regex(&self) -> &Regex { + &self.regex + } + + pub fn substitution_string(&self) -> &str { + &self.substitution_string + } + + pub fn guard(&self) -> &QueryGuard { + &self.guard + } + + pub fn storage_type(&self) -> &StorageType { + &self.storage_type + } + + pub fn match_formats(&self) -> &Vec { + &self.guard.match_formats + } + + pub fn match_class(&self) -> &Vec { + &self.guard.match_class + } + + pub fn match_reference_name(&self) -> &Regex { + &self.guard.match_reference_name + } + + pub fn start_interval(&self) -> Interval { + self.guard.start_interval + } + + pub fn end_interval(&self) -> Interval { + self.guard.end_interval + } + + pub fn match_fields(&self) -> &Fields { + &self.guard.match_fields + } + + pub fn match_tags(&self) -> &Tags { + &self.guard.match_tags + } + + pub fn match_no_tags(&self) -> &NoTags { + &self.guard.match_no_tags + } } impl IdResolver for RegexResolver { From f4b4aa4e478fa69d36cf7554b539897e880bc379 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 12 Dec 2022 17:31:17 +1100 Subject: [PATCH 22/45] config: add cors tests and environment variable tests --- htsget-config/Cargo.toml | 5 +- htsget-config/config.toml | 2 +- htsget-config/src/config/cors.rs | 98 ++++++-- htsget-config/src/config/mod.rs | 283 +++++++++++++++++------- htsget-config/src/regex_resolver/mod.rs | 6 +- 5 files changed, 287 insertions(+), 107 deletions(-) diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index d556bc33d..63e47f47c 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -20,4 +20,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] } toml = "0.5" http = "0.2" -http-serde = "1.1" \ No newline at end of file +http-serde = "1.1" + +[dev-dependencies] +figment = { version = "0.10", features = ["test"] } \ No newline at end of file diff --git a/htsget-config/config.toml b/htsget-config/config.toml index 683c9b486..e74a53fdf 100644 --- a/htsget-config/config.toml +++ b/htsget-config/config.toml @@ -5,7 +5,7 @@ ticket_server_addr = "127.0.0.1:8082" #data_server_path = "data" #data_server_serve_at = "/data" #data_server_config = "None" -data_server_config = [] +#data_server_config = [] #data_server_addr = "127.0.0.1:8082" #data_server_cors_allow_credentials = false #data_server_cors_allow_origins = ["http://localhost:8081"] diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs index a70ff61fb..7dcd32491 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/cors.rs @@ -11,7 +11,7 @@ use crate::config::default_server_origin; const CORS_MAX_AGE: usize = 86400; /// Tagged allow headers for cors config. Either Mirror or Any. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub enum TaggedAllowTypes { #[serde(alias = "mirror", alias = "MIRROR")] Mirror, @@ -20,7 +20,7 @@ pub enum TaggedAllowTypes { } /// Tagged allow headers for cors config. Either Mirror or Any. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub enum TaggedAnyAllowType { #[serde(alias = "any", alias = "ANY")] Any @@ -28,7 +28,7 @@ pub enum TaggedAnyAllowType { /// Allowed header for cors config. Any allows all headers by sending a wildcard, /// and mirror allows all headers by mirroring the received headers. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(untagged)] pub enum AllowType { Tagged(Tagged), @@ -59,7 +59,7 @@ fn deserialize_allow_types<'de, D, T>(deserializer: D) -> Result, D::Erro names.into_iter().map(|name| T::from_str(&name).map_err(Error::custom)).collect() } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct HeaderValue(HeaderValueInner); impl FromStr for HeaderValue { @@ -80,49 +80,103 @@ impl Display for HeaderValue { #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct CorsConfig { - cors_allow_credentials: bool, - cors_allow_origins: AllowType, - cors_allow_headers: AllowType, - cors_allow_methods: AllowType, - cors_max_age: usize, - cors_expose_headers: AllowType, + allow_credentials: bool, + allow_origins: AllowType, + allow_headers: AllowType, + allow_methods: AllowType, + max_age: usize, + expose_headers: AllowType, } impl CorsConfig { pub fn allow_credentials(&self) -> bool { - self.cors_allow_credentials + self.allow_credentials } pub fn allow_origins(&self) -> &AllowType { - &self.cors_allow_origins + &self.allow_origins } pub fn allow_headers(&self) -> &AllowType { - &self.cors_allow_headers + &self.allow_headers } pub fn allow_methods(&self) -> &AllowType { - &self.cors_allow_methods + &self.allow_methods } pub fn max_age(&self) -> usize { - self.cors_max_age + self.max_age } pub fn expose_headers(&self) -> &AllowType { - &self.cors_expose_headers + &self.expose_headers } } impl Default for CorsConfig { fn default() -> Self { Self { - cors_allow_credentials: false, - cors_allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static(default_server_origin()))]), - cors_allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), - cors_allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), - cors_max_age: CORS_MAX_AGE, - cors_expose_headers: AllowType::List(vec![]), + allow_credentials: false, + allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static(default_server_origin()))]), + allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), + allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), + max_age: CORS_MAX_AGE, + expose_headers: AllowType::List(vec![]), } } +} + +mod tests { + use std::fmt::Debug; + use http::Method; + use serde::Deserialize; + use toml::de::Error; + use crate::config::cors::{AllowType, CorsConfig, TaggedAllowTypes, TaggedAnyAllowType}; + + fn test_cors_config(input: &str, expected: &T, get_result: F) + where F: Fn(&CorsConfig) -> &T, + T: Debug + Eq { + let config: CorsConfig = toml::from_str(input).unwrap(); + assert_eq!(expected, get_result(&config)); + + let serialized = toml::to_string(&config).unwrap(); + let deserialized = toml::from_str(&serialized).unwrap(); + assert_eq!(expected, get_result(&deserialized)); + } + + #[test] + fn unit_variant_any_allow_type() { + test_cors_config("cors_allow_methods = \"Any\"", + &AllowType::Tagged(TaggedAllowTypes::Any), + |config| config.allow_methods()); + } + + #[test] + fn unit_variant_mirror_allow_type() { + test_cors_config("cors_allow_methods = \"Mirror\"", + &AllowType::Tagged(TaggedAllowTypes::Mirror), + |config| config.allow_methods()); + } + + #[test] + fn list_allow_type() { + test_cors_config("cors_allow_methods = [\"GET\"]", + &AllowType::List(vec![Method::GET]), + |config| config.allow_methods()); + } + + #[test] + fn tagged_any_allow_type() { + test_cors_config("cors_expose_headers = \"Any\"", + &AllowType::Tagged(TaggedAnyAllowType::Any), + |config| config.expose_headers()); + } + + #[test] + fn tagged_any_allow_type_err_on_mirror() { + let allow_type_method = "cors_expose_headers = \"Mirror\""; + let config: Result = toml::from_str(allow_type_method); + assert!(matches!(config, Err(_))); + } } \ No newline at end of file diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 52ba10759..8afd41608 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -17,6 +17,7 @@ use http::{HeaderValue as HeaderValueInner, Method}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde::de::Error as DeError; use serde::ser::Error as SeError; +use serde_with::with_prefix; use tracing::info; use tracing::instrument; use tracing_subscriber::layer::SubscriberExt; @@ -98,16 +99,18 @@ struct Args { pub struct Config { #[serde(flatten)] ticket_server: TicketServerConfig, - data_server: Vec, + data_servers: Vec, resolvers: Vec, } +with_prefix!(ticket_server_prefix "ticket_server_"); + /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { ticket_server_addr: SocketAddr, - #[serde(flatten)] + #[serde(flatten, with = "ticket_server_prefix")] cors: CorsConfig, #[serde(flatten)] service_info: ServiceInfo, @@ -339,7 +342,7 @@ impl Default for Config { fn default() -> Self { Self { ticket_server: TicketServerConfig::default(), - data_server: vec![DataServerConfig::default()], + data_servers: vec![DataServerConfig::default()], resolvers: vec![RegexResolver::default(), RegexResolver::default()], } } @@ -356,7 +359,7 @@ impl Config { pub fn from_env(config: PathBuf) -> io::Result { let config = Figment::from(Serialized::defaults(Config::default())) .merge(Toml::file(config)) - .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX).split("_")) + .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX)) .extract() .map_err(|err| { io::Error::new(ErrorKind::Other, format!("failed to parse config: {}", err)) @@ -387,8 +390,8 @@ impl Config { &self.ticket_server } - pub fn data_server(&self) -> &Vec { - &self.data_server + pub fn data_servers(&self) -> &Vec { + &self.data_servers } pub fn resolvers(&self) -> &Vec { @@ -398,80 +401,200 @@ impl Config { #[cfg(test)] mod tests { + use figment::Jail; + use regex::Regex; + use crate::config::cors::AllowType::Tagged; + use crate::Format::Bam; + use crate::regex_resolver::{Scheme, StorageType}; use super::*; - // #[test] - // fn config_addr() { - // std::env::set_var("HTSGET_TICKET_SERVER_ADDR", "127.0.0.1:8081"); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!( - // config.ticket_server_config.addr, - // "127.0.0.1:8081".parse().unwrap() - // ); - // } - - // #[test] - // fn config_ticket_server_cors_allow_origin() { - // std::env::set_var( - // "HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGIN", - // "http://localhost:8080", - // ); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!( - // config.ticket_server_config.cors_allow_origin, - // "http://localhost:8080" - // ); - // } - - // #[test] - // fn config_data_server_cors_allow_origin() { - // std::env::set_var( - // "HTSGET_DATA_SERVER_CORS_ALLOW_ORIGIN", - // "http://localhost:8080", - // ); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!( - // config.data_server_config.data_server_cors_allow_origin, - // "http://localhost:8080" - // ); - // } - // - // #[test] - // fn config_ticket_server_addr() { - // std::env::set_var("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082"); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!( - // config.data_server_config.data_server_addr, - // "127.0.0.1:8082".parse().unwrap() - // ); - // } - // - // #[test] - // fn config_regex() { - // std::env::set_var("HTSGET_REGEX", ".+"); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!(config.resolver.regex.to_string(), ".+"); - // } - // - // #[test] - // fn config_substitution_string() { - // std::env::set_var("HTSGET_SUBSTITUTION_STRING", "$0-test"); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!(config.resolver.substitution_string, "$0-test"); - // } - - // #[test] - // fn config_service_info_id() { - // std::env::set_var("HTSGET_ID", "id"); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!(config.ticket_server_config.service_info.id.unwrap(), "id"); - // } - - // #[cfg(feature = "s3-storage")] - // #[test] - // fn config_storage_type() { - // std::env::set_var("HTSGET_STORAGE_TYPE", "AwsS3Storage"); - // let config = Config::from_env(PathBuf::from("config.toml")).unwrap(); - // assert_eq!(config.storage_type, StorageType::AwsS3Storage); - // } + fn test_config(contents: Option<&str>, env_variables: Vec<(K, V)>, test_fn: F) + where + K: AsRef, + V: Display, + F: FnOnce(Config) { + Jail::expect_with(|jail| { + if let Some(contents) = contents { + jail.create_file("test.toml", contents)?; + } + + for (key, value) in env_variables { + jail.set_env(key, value); + } + + test_fn(Config::from_env("test.toml".into()).map_err(|err| err.to_string())?); + + Ok(()) + }); + } + + fn test_config_from_env(env_variables: Vec<(K, V)>, test_fn: F) + where + K: AsRef, + V: Display, + F: FnOnce(Config) { + test_config(None, env_variables, test_fn); + } + + fn test_config_from_file(contents: &str, test_fn: F) + where + F: FnOnce(Config) { + test_config(Some(contents), Vec::<(&str, &str)>::new(), test_fn); + } + + #[test] + fn config_ticket_server_addr_env() { + test_config_from_env(vec![("HTSGET_TICKET_SERVER_ADDR", "127.0.0.1:8082")], |config| { + assert_eq!( + config.ticket_server().addr(), + "127.0.0.1:8082".parse().unwrap() + ); + }); + } + + #[test] + fn config_ticket_server_cors_allow_origin_env() { + test_config_from_env(vec![("HTSGET_TICKET_SERVER_ALLOW_CREDENTIALS", true)], |config| { + assert!( + config.ticket_server().allow_credentials() + ); + }); + } + + #[test] + fn config_service_info_id_env() { + test_config_from_env(vec![("HTSGET_ID", "id")], |config| { + assert_eq!( + config.ticket_server().id(), + &Some("id".to_string()) + ); + }); + } + + #[test] + fn config_data_server_addr_env() { + test_config_from_env( vec![("HTSGET_DATA_SERVERS", "[{addr=127.0.0.1:8082}]")], |config| { + assert_eq!( + config.data_servers().first().unwrap().addr(), + "127.0.0.1:8082".parse().unwrap() + ); + }); + } + + #[test] + fn config_resolvers_env() { + test_config_from_env(vec![("HTSGET_RESOLVERS", "[{regex=regex}]")], |config| { + assert_eq!( + config.resolvers().first().unwrap().regex().as_str(), + "regex" + ); + }); + } + + #[test] + fn config_ticket_server_addr_file() { + test_config_from_file(r#"ticket_server_addr = "127.0.0.1:8082""#, |config| { + assert_eq!( + config.ticket_server().addr(), + "127.0.0.1:8082".parse().unwrap() + ); + }); + } + + #[test] + fn config_ticket_server_cors_allow_origin_file() { + test_config_from_file(r#"ticket_server_allow_credentials = true"#, |config| { + assert!( + config.ticket_server().allow_credentials() + ); + }); + } + + #[test] + fn config_service_info_id_file() { + test_config_from_file(r#"id = "id""#, |config| { + assert_eq!( + config.ticket_server().id(), + &Some("id".to_string()) + ); + }); + } + + #[test] + fn config_data_server_addr_file() { + test_config_from_file(r#" + [[data_servers]] + addr = "127.0.0.1:8082" + "#, |config| { + assert_eq!( + config.data_servers().first().unwrap().addr(), + "127.0.0.1:8082".parse().unwrap() + ); + }); + } + + #[test] + fn config_resolvers_file() { + test_config_from_file(r#" + [[resolvers]] + regex = "regex" + "#, |config| { + assert_eq!( + config.resolvers().first().unwrap().regex().as_str(), + "regex" + ); + }); + } + + #[test] + fn config_resolvers_guard_file() { + test_config_from_file(r#" + [[resolvers]] + regex = "regex" + + [resolvers.guard] + match_formats = ["BAM"] + "#, |config| { + assert_eq!( + config.resolvers().first().unwrap().match_formats(), + &vec![Bam] + ); + }); + } + + #[test] + fn config_storage_type_url_file() { + test_config_from_file(r#" + [[resolvers]] + regex = "regex" + + [resolvers.storage_type] + type = "Url" + path = "path" + scheme = "HTTPS" + "#, |config| { + assert!(matches!( + config.resolvers().first().unwrap().storage_type(), + StorageType::Url(resolver) if resolver.path() == "path" && resolver.scheme() == Scheme::Https + )); + }); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn config_storage_type_s3_file() { + test_config_from_file(r#" + [[resolvers]] + regex = "regex" + + [resolvers.storage_type] + type = "S3" + bucket = "bucket" + "#, |config| { + assert!(matches!( + config.resolvers().first().unwrap().storage_type(), + StorageType::S3(resolver) if resolver.bucket() == "bucket" + )); + }); + } } diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 6b6d2b013..0df656d61 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -43,7 +43,7 @@ impl Default for StorageType { } } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] pub enum Scheme { #[serde(alias = "http", alias = "HTTP")] Http, @@ -68,8 +68,8 @@ pub struct UrlResolver { } impl UrlResolver { - pub fn scheme(&self) -> &Scheme { - &self.scheme + pub fn scheme(&self) -> Scheme { + self.scheme } pub fn authority(&self) -> &Authority { From 85f522b7d9bf61bb83c94ae80402b42b303e2678 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 16 Dec 2022 14:52:03 +1100 Subject: [PATCH 23/45] config: deserialize empty string as None value --- htsget-config/config.toml | 1 + htsget-config/src/config/cors.rs | 262 ++++---- htsget-config/src/config/mod.rs | 853 ++++++++++++------------ htsget-config/src/regex_resolver/mod.rs | 458 ++++++------- htsget-search/src/lib.rs | 2 +- htsget-search/src/storage/mod.rs | 2 +- 6 files changed, 820 insertions(+), 758 deletions(-) diff --git a/htsget-config/config.toml b/htsget-config/config.toml index e74a53fdf..cdbbe6d49 100644 --- a/htsget-config/config.toml +++ b/htsget-config/config.toml @@ -1,4 +1,5 @@ ticket_server_addr = "127.0.0.1:8082" +data_server = "None" #ticket_server_cors_allow_credentials = false #ticket_server_cors_allow_origin = "http://localhost:8080" #start_data_server = true diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs index 7dcd32491..b49777ddf 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/cors.rs @@ -1,11 +1,11 @@ -use std::fmt::{Display, Formatter}; -use std::str::FromStr; -use http::header::{HeaderName, InvalidHeaderValue, HeaderValue as HeaderValueInner}; +use crate::config::default_server_origin; +use http::header::{HeaderName, HeaderValue as HeaderValueInner, InvalidHeaderValue}; use http::Method; -use serde::{Deserialize, Serialize, Deserializer, Serializer}; use serde::de::Error; use serde::ser::SerializeSeq; -use crate::config::default_server_origin; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt::{Display, Formatter}; +use std::str::FromStr; /// The maximum default amount of time a CORS request can be cached for in seconds. const CORS_MAX_AGE: usize = 86400; @@ -13,17 +13,17 @@ const CORS_MAX_AGE: usize = 86400; /// Tagged allow headers for cors config. Either Mirror or Any. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub enum TaggedAllowTypes { - #[serde(alias = "mirror", alias = "MIRROR")] - Mirror, - #[serde(alias = "any", alias = "ANY")] - Any + #[serde(alias = "mirror", alias = "MIRROR")] + Mirror, + #[serde(alias = "any", alias = "ANY")] + Any, } /// Tagged allow headers for cors config. Either Mirror or Any. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub enum TaggedAnyAllowType { - #[serde(alias = "any", alias = "ANY")] - Any + #[serde(alias = "any", alias = "ANY")] + Any, } /// Allowed header for cors config. Any allows all headers by sending a wildcard, @@ -31,152 +31,170 @@ pub enum TaggedAnyAllowType { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(untagged)] pub enum AllowType { - Tagged(Tagged), - #[serde(bound(serialize = "T: Display", deserialize = "T: FromStr, T::Err: Display"))] - #[serde(serialize_with = "serialize_allow_types", deserialize_with = "deserialize_allow_types")] - List(Vec) + Tagged(Tagged), + #[serde(bound(serialize = "T: Display", deserialize = "T: FromStr, T::Err: Display"))] + #[serde( + serialize_with = "serialize_allow_types", + deserialize_with = "deserialize_allow_types" + )] + List(Vec), } fn serialize_allow_types(names: &Vec, serializer: S) -> Result - where - T: Display, - S: Serializer +where + T: Display, + S: Serializer, { - let mut sequence = serializer.serialize_seq(Some(names.len()))?; - for element in names.iter().map(|name| format!("{}", name)) { - sequence.serialize_element(&element)?; - } - sequence.end() + let mut sequence = serializer.serialize_seq(Some(names.len()))?; + for element in names.iter().map(|name| format!("{}", name)) { + sequence.serialize_element(&element)?; + } + sequence.end() } fn deserialize_allow_types<'de, D, T>(deserializer: D) -> Result, D::Error> - where - T: FromStr, - T::Err: Display, - D: Deserializer<'de> +where + T: FromStr, + T::Err: Display, + D: Deserializer<'de>, { - let names: Vec = Deserialize::deserialize(deserializer)?; - names.into_iter().map(|name| T::from_str(&name).map_err(Error::custom)).collect() + let names: Vec = Deserialize::deserialize(deserializer)?; + names + .into_iter() + .map(|name| T::from_str(&name).map_err(Error::custom)) + .collect() } #[derive(Debug, Clone, PartialEq, Eq)] pub struct HeaderValue(HeaderValueInner); impl FromStr for HeaderValue { - type Err = InvalidHeaderValue; + type Err = InvalidHeaderValue; - fn from_str(header: &str) -> Result { - Ok(HeaderValue(HeaderValueInner::from_str(header)?)) - } + fn from_str(header: &str) -> Result { + Ok(HeaderValue(HeaderValueInner::from_str(header)?)) + } } impl Display for HeaderValue { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&String::from_utf8_lossy(self.0.as_ref())) - } + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&String::from_utf8_lossy(self.0.as_ref())) + } } /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct CorsConfig { - allow_credentials: bool, - allow_origins: AllowType, - allow_headers: AllowType, - allow_methods: AllowType, - max_age: usize, - expose_headers: AllowType, + allow_credentials: bool, + allow_origins: AllowType, + allow_headers: AllowType, + allow_methods: AllowType, + max_age: usize, + expose_headers: AllowType, } impl CorsConfig { - pub fn allow_credentials(&self) -> bool { - self.allow_credentials - } + pub fn allow_credentials(&self) -> bool { + self.allow_credentials + } - pub fn allow_origins(&self) -> &AllowType { - &self.allow_origins - } + pub fn allow_origins(&self) -> &AllowType { + &self.allow_origins + } - pub fn allow_headers(&self) -> &AllowType { - &self.allow_headers - } + pub fn allow_headers(&self) -> &AllowType { + &self.allow_headers + } - pub fn allow_methods(&self) -> &AllowType { - &self.allow_methods - } + pub fn allow_methods(&self) -> &AllowType { + &self.allow_methods + } - pub fn max_age(&self) -> usize { - self.max_age - } + pub fn max_age(&self) -> usize { + self.max_age + } - pub fn expose_headers(&self) -> &AllowType { - &self.expose_headers - } + pub fn expose_headers(&self) -> &AllowType { + &self.expose_headers + } } impl Default for CorsConfig { - fn default() -> Self { - Self { - allow_credentials: false, - allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static(default_server_origin()))]), - allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), - allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), - max_age: CORS_MAX_AGE, - expose_headers: AllowType::List(vec![]), - } - } + fn default() -> Self { + Self { + allow_credentials: false, + allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static( + default_server_origin(), + ))]), + allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), + allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), + max_age: CORS_MAX_AGE, + expose_headers: AllowType::List(vec![]), + } + } } +#[cfg(test)] mod tests { - use std::fmt::Debug; - use http::Method; - use serde::Deserialize; - use toml::de::Error; - use crate::config::cors::{AllowType, CorsConfig, TaggedAllowTypes, TaggedAnyAllowType}; - - fn test_cors_config(input: &str, expected: &T, get_result: F) - where F: Fn(&CorsConfig) -> &T, - T: Debug + Eq { - let config: CorsConfig = toml::from_str(input).unwrap(); - assert_eq!(expected, get_result(&config)); - - let serialized = toml::to_string(&config).unwrap(); - let deserialized = toml::from_str(&serialized).unwrap(); - assert_eq!(expected, get_result(&deserialized)); - } - - #[test] - fn unit_variant_any_allow_type() { - test_cors_config("cors_allow_methods = \"Any\"", - &AllowType::Tagged(TaggedAllowTypes::Any), - |config| config.allow_methods()); - } - - #[test] - fn unit_variant_mirror_allow_type() { - test_cors_config("cors_allow_methods = \"Mirror\"", - &AllowType::Tagged(TaggedAllowTypes::Mirror), - |config| config.allow_methods()); - } - - #[test] - fn list_allow_type() { - test_cors_config("cors_allow_methods = [\"GET\"]", - &AllowType::List(vec![Method::GET]), - |config| config.allow_methods()); - } - - #[test] - fn tagged_any_allow_type() { - test_cors_config("cors_expose_headers = \"Any\"", - &AllowType::Tagged(TaggedAnyAllowType::Any), - |config| config.expose_headers()); - } - - #[test] - fn tagged_any_allow_type_err_on_mirror() { - let allow_type_method = "cors_expose_headers = \"Mirror\""; - let config: Result = toml::from_str(allow_type_method); - assert!(matches!(config, Err(_))); - } -} \ No newline at end of file + use super::*; + use http::Method; + use std::fmt::Debug; + use toml::de::Error; + + fn test_cors_config(input: &str, expected: &T, get_result: F) + where + F: Fn(&CorsConfig) -> &T, + T: Debug + Eq, + { + let config: CorsConfig = toml::from_str(input).unwrap(); + assert_eq!(expected, get_result(&config)); + + let serialized = toml::to_string(&config).unwrap(); + let deserialized = toml::from_str(&serialized).unwrap(); + assert_eq!(expected, get_result(&deserialized)); + } + + #[test] + fn unit_variant_any_allow_type() { + test_cors_config( + "cors_allow_methods = \"Any\"", + &AllowType::Tagged(TaggedAllowTypes::Any), + |config| config.allow_methods(), + ); + } + + #[test] + fn unit_variant_mirror_allow_type() { + test_cors_config( + "cors_allow_methods = \"Mirror\"", + &AllowType::Tagged(TaggedAllowTypes::Mirror), + |config| config.allow_methods(), + ); + } + + #[test] + fn list_allow_type() { + test_cors_config( + "cors_allow_methods = [\"GET\"]", + &AllowType::List(vec![Method::GET]), + |config| config.allow_methods(), + ); + } + + #[test] + fn tagged_any_allow_type() { + test_cors_config( + "cors_expose_headers = \"Any\"", + &AllowType::Tagged(TaggedAnyAllowType::Any), + |config| config.expose_headers(), + ); + } + + #[test] + fn tagged_any_allow_type_err_on_mirror() { + let allow_type_method = "cors_expose_headers = \"Mirror\""; + let config: Result = toml::from_str(allow_type_method); + assert!(matches!(config, Err(_))); + } +} diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 8afd41608..67b3076fb 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -1,28 +1,25 @@ pub mod cors; -use std::fmt::{Debug, Display, Formatter}; +use std::fmt::Debug; use std::io; use std::io::ErrorKind; use std::net::SocketAddr; use std::path::PathBuf; -use std::str::FromStr; -use std::time::Duration; -use crate::regex_resolver::aws::S3Resolver; +use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAnyAllowType}; use clap::Parser; use figment::providers::{Env, Format, Serialized, Toml}; use figment::Figment; -use http::header::{HeaderName, InvalidHeaderValue}; -use http::{HeaderValue as HeaderValueInner, Method}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use serde::de::Error as DeError; -use serde::ser::Error as SeError; +use http::header::HeaderName; +use http::Method; +use regex::internal::Input; +use serde::{de, Deserialize, Deserializer, Serialize}; +use serde::de::IntoDeserializer; use serde_with::with_prefix; use tracing::info; use tracing::instrument; use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::{EnvFilter, fmt, Registry}; -use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAnyAllowType}; +use tracing_subscriber::{fmt, EnvFilter, Registry}; use crate::regex_resolver::RegexResolver; @@ -66,41 +63,57 @@ The next variables are used to configure the info for the service-info endpoints const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET_"; pub(crate) fn default_localstorage_addr() -> &'static str { - "127.0.0.1:8081" + "127.0.0.1:8081" } fn default_addr() -> &'static str { - "127.0.0.1:8080" + "127.0.0.1:8080" } fn default_server_origin() -> &'static str { - "http://localhost:8080" + "http://localhost:8080" } fn default_path() -> &'static str { - "data" + "data" } pub(crate) fn default_serve_at() -> &'static str { - "/data" + "/data" } /// The command line arguments allowed for the htsget-rs executables. #[derive(Parser, Debug)] #[command(author, version, about, long_about = USAGE)] struct Args { - #[arg(short, long, env = "HTSGET_CONFIG")] - config: PathBuf, + #[arg(short, long, env = "HTSGET_CONFIG")] + config: PathBuf, +} + +fn empty_string_as_none<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de> +{ + let optional_string = Option::deserialize(deserializer)?.filter(|s: &String| !s.is_empty() && s.to_lowercase() != "none"); + if let Some(string) = optional_string { + Ok(Some( + T::deserialize(string.into_deserializer())?, + )) + } else { + Ok(None) + } } /// Configuration for the server. Each field will be read from environment variables. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct Config { - #[serde(flatten)] - ticket_server: TicketServerConfig, - data_servers: Vec, - resolvers: Vec, + #[serde(flatten)] + ticket_server: TicketServerConfig, + #[serde(deserialize_with = "empty_string_as_none")] + data_server: Option, + resolvers: Vec, } with_prefix!(ticket_server_prefix "ticket_server_"); @@ -109,462 +122,475 @@ with_prefix!(ticket_server_prefix "ticket_server_"); #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { - ticket_server_addr: SocketAddr, - #[serde(flatten, with = "ticket_server_prefix")] - cors: CorsConfig, - #[serde(flatten)] - service_info: ServiceInfo, + ticket_server_addr: SocketAddr, + #[serde(flatten, with = "ticket_server_prefix")] + cors: CorsConfig, + #[serde(flatten)] + service_info: ServiceInfo, } impl TicketServerConfig { - pub fn addr(&self) -> SocketAddr { - self.ticket_server_addr - } + pub fn addr(&self) -> SocketAddr { + self.ticket_server_addr + } - pub fn cors(&self) -> &CorsConfig { - &self.cors - } + pub fn cors(&self) -> &CorsConfig { + &self.cors + } - pub fn service_info(&self) -> &ServiceInfo { - &self.service_info - } + pub fn service_info(&self) -> &ServiceInfo { + &self.service_info + } - pub fn allow_credentials(&self) -> bool { - self.cors.allow_credentials() - } + pub fn allow_credentials(&self) -> bool { + self.cors.allow_credentials() + } - pub fn allow_origins(&self) -> &AllowType { - self.cors.allow_origins() - } + pub fn allow_origins(&self) -> &AllowType { + self.cors.allow_origins() + } - pub fn allow_headers(&self) -> &AllowType { - self.cors.allow_headers() - } + pub fn allow_headers(&self) -> &AllowType { + self.cors.allow_headers() + } - pub fn allow_methods(&self) -> &AllowType { - self.cors.allow_methods() - } + pub fn allow_methods(&self) -> &AllowType { + self.cors.allow_methods() + } - pub fn max_age(&self) -> usize { - self.cors.max_age() - } + pub fn max_age(&self) -> usize { + self.cors.max_age() + } - pub fn expose_headers(&self) -> &AllowType { - self.cors.expose_headers() - } + pub fn expose_headers(&self) -> &AllowType { + self.cors.expose_headers() + } - pub fn id(&self) -> &Option { - self.service_info.id() - } + pub fn id(&self) -> &Option { + self.service_info.id() + } - pub fn name(&self) -> &Option { - self.service_info.name() - } + pub fn name(&self) -> &Option { + self.service_info.name() + } - pub fn version(&self) -> &Option { - self.service_info.version() - } + pub fn version(&self) -> &Option { + self.service_info.version() + } - pub fn organization_name(&self) -> &Option { - self.service_info.organization_name() - } + pub fn organization_name(&self) -> &Option { + self.service_info.organization_name() + } - pub fn organization_url(&self) -> &Option { - self.service_info.organization_url() - } + pub fn organization_url(&self) -> &Option { + self.service_info.organization_url() + } - pub fn contact_url(&self) -> &Option { - self.service_info.contact_url() - } + pub fn contact_url(&self) -> &Option { + self.service_info.contact_url() + } - pub fn documentation_url(&self) -> &Option { - self.service_info.documentation_url() - } + pub fn documentation_url(&self) -> &Option { + self.service_info.documentation_url() + } - pub fn created_at(&self) -> &Option { - self.service_info.created_at() - } + pub fn created_at(&self) -> &Option { + self.service_info.created_at() + } - pub fn updated_at(&self) -> &Option { - self.service_info.updated_at() - } + pub fn updated_at(&self) -> &Option { + self.service_info.updated_at() + } - pub fn environment(&self) -> &Option { - self.service_info.environment() - } + pub fn environment(&self) -> &Option { + self.service_info.environment() + } } /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct DataServerConfig { - addr: SocketAddr, - path: PathBuf, - serve_at: PathBuf, - key: Option, - cert: Option, - #[serde(flatten)] - cors: CorsConfig, + addr: SocketAddr, + path: PathBuf, + serve_at: PathBuf, + key: Option, + cert: Option, + #[serde(flatten)] + cors: CorsConfig, } impl DataServerConfig { - pub fn addr(&self) -> SocketAddr { - self.addr - } + pub fn addr(&self) -> SocketAddr { + self.addr + } - pub fn path(&self) -> &PathBuf { - &self.path - } + pub fn path(&self) -> &PathBuf { + &self.path + } - pub fn serve_at(&self) -> &PathBuf { - &self.serve_at - } + pub fn serve_at(&self) -> &PathBuf { + &self.serve_at + } - pub fn key(&self) -> &Option { - &self.key - } + pub fn key(&self) -> &Option { + &self.key + } - pub fn cert(&self) -> &Option { - &self.cert - } + pub fn cert(&self) -> &Option { + &self.cert + } - pub fn cors(&self) -> &CorsConfig { - &self.cors - } + pub fn cors(&self) -> &CorsConfig { + &self.cors + } - pub fn allow_credentials(&self) -> bool { - self.cors.allow_credentials() - } + pub fn allow_credentials(&self) -> bool { + self.cors.allow_credentials() + } - pub fn allow_origins(&self) -> &AllowType { - self.cors.allow_origins() - } + pub fn allow_origins(&self) -> &AllowType { + self.cors.allow_origins() + } - pub fn allow_headers(&self) -> &AllowType { - self.cors.allow_headers() - } + pub fn allow_headers(&self) -> &AllowType { + self.cors.allow_headers() + } - pub fn allow_methods(&self) -> &AllowType { - self.cors.allow_methods() - } + pub fn allow_methods(&self) -> &AllowType { + self.cors.allow_methods() + } - pub fn max_age(&self) -> usize { - self.cors.max_age() - } + pub fn max_age(&self) -> usize { + self.cors.max_age() + } - pub fn expose_headers(&self) -> &AllowType { - self.cors.expose_headers() - } + pub fn expose_headers(&self) -> &AllowType { + self.cors.expose_headers() + } } impl Default for DataServerConfig { - fn default() -> Self { - Self { - addr: default_localstorage_addr().parse().expect("expected valid address"), - path: default_path().into(), - serve_at: default_serve_at().into(), - key: None, - cert: None, - cors: CorsConfig::default(), - } - } + fn default() -> Self { + Self { + addr: default_localstorage_addr() + .parse() + .expect("expected valid address"), + path: default_path().into(), + serve_at: default_serve_at().into(), + key: None, + cert: None, + cors: CorsConfig::default(), + } + } } /// Configuration of the service info. #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct ServiceInfo { - id: Option, - name: Option, - version: Option, - organization_name: Option, - organization_url: Option, - contact_url: Option, - documentation_url: Option, - created_at: Option, - updated_at: Option, - environment: Option, + id: Option, + name: Option, + version: Option, + organization_name: Option, + organization_url: Option, + contact_url: Option, + documentation_url: Option, + created_at: Option, + updated_at: Option, + environment: Option, } impl ServiceInfo { - pub fn id(&self) -> &Option { - &self.id - } + pub fn id(&self) -> &Option { + &self.id + } - pub fn name(&self) -> &Option { - &self.name - } + pub fn name(&self) -> &Option { + &self.name + } - pub fn version(&self) -> &Option { - &self.version - } + pub fn version(&self) -> &Option { + &self.version + } - pub fn organization_name(&self) -> &Option { - &self.organization_name - } + pub fn organization_name(&self) -> &Option { + &self.organization_name + } - pub fn organization_url(&self) -> &Option { - &self.organization_url - } + pub fn organization_url(&self) -> &Option { + &self.organization_url + } - pub fn contact_url(&self) -> &Option { - &self.contact_url - } + pub fn contact_url(&self) -> &Option { + &self.contact_url + } - pub fn documentation_url(&self) -> &Option { - &self.documentation_url - } + pub fn documentation_url(&self) -> &Option { + &self.documentation_url + } - pub fn created_at(&self) -> &Option { - &self.created_at - } + pub fn created_at(&self) -> &Option { + &self.created_at + } - pub fn updated_at(&self) -> &Option { - &self.updated_at - } + pub fn updated_at(&self) -> &Option { + &self.updated_at + } - pub fn environment(&self) -> &Option { - &self.environment - } + pub fn environment(&self) -> &Option { + &self.environment + } } impl Default for TicketServerConfig { - fn default() -> Self { - Self { - ticket_server_addr: default_addr().parse().expect("expected valid address"), - cors: CorsConfig::default(), - service_info: ServiceInfo::default(), - } + fn default() -> Self { + Self { + ticket_server_addr: default_addr().parse().expect("expected valid address"), + cors: CorsConfig::default(), + service_info: ServiceInfo::default(), } + } } impl Default for Config { - fn default() -> Self { - Self { - ticket_server: TicketServerConfig::default(), - data_servers: vec![DataServerConfig::default()], - resolvers: vec![RegexResolver::default(), RegexResolver::default()], - } + fn default() -> Self { + Self { + ticket_server: TicketServerConfig::default(), + data_server: Some(DataServerConfig::default()), + resolvers: vec![RegexResolver::default(), RegexResolver::default()], } + } } impl Config { - /// Parse the command line arguments - pub fn parse_args() -> PathBuf { - Args::parse().config - } - - /// Read the environment variables into a Config struct. - #[instrument] - pub fn from_env(config: PathBuf) -> io::Result { - let config = Figment::from(Serialized::defaults(Config::default())) - .merge(Toml::file(config)) - .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX)) - .extract() - .map_err(|err| { - io::Error::new(ErrorKind::Other, format!("failed to parse config: {}", err)) - })?; - - info!(config = ?config, "config created from environment variables"); - Ok(config) - } - - /// Setup tracing, using a global subscriber. - pub fn setup_tracing() -> io::Result<()> { - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - let fmt_layer = fmt::Layer::default(); - - let subscriber = Registry::default().with(env_filter).with(fmt_layer); - - tracing::subscriber::set_global_default(subscriber).map_err(|err| { - io::Error::new( - ErrorKind::Other, - format!("failed to install `tracing` subscriber: {}", err), - ) - })?; - - Ok(()) - } - - pub fn ticket_server(&self) -> &TicketServerConfig { - &self.ticket_server - } - - pub fn data_servers(&self) -> &Vec { - &self.data_servers - } - - pub fn resolvers(&self) -> &Vec { - &self.resolvers - } + /// Parse the command line arguments + pub fn parse_args() -> PathBuf { + Args::parse().config + } + + /// Read the environment variables into a Config struct. + #[instrument] + pub fn from_env(config: PathBuf) -> io::Result { + let config = Figment::from(Serialized::defaults(Config::default())) + .merge(Toml::file(config)) + .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX)) + .extract() + .map_err(|err| { + io::Error::new(ErrorKind::Other, format!("failed to parse config: {}", err)) + })?; + + info!(config = ?config, "config created from environment variables"); + Ok(config) + } + + /// Setup tracing, using a global subscriber. + pub fn setup_tracing() -> io::Result<()> { + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + let fmt_layer = fmt::Layer::default(); + + let subscriber = Registry::default().with(env_filter).with(fmt_layer); + + tracing::subscriber::set_global_default(subscriber).map_err(|err| { + io::Error::new( + ErrorKind::Other, + format!("failed to install `tracing` subscriber: {}", err), + ) + })?; + + Ok(()) + } + + pub fn ticket_server(&self) -> &TicketServerConfig { + &self.ticket_server + } + + pub fn data_server(&self) -> Option<&DataServerConfig> { + self.data_server.as_ref() + } + + pub fn resolvers(&self) -> &Vec { + &self.resolvers + } } #[cfg(test)] mod tests { - use figment::Jail; - use regex::Regex; - use crate::config::cors::AllowType::Tagged; - use crate::Format::Bam; - use crate::regex_resolver::{Scheme, StorageType}; - use super::*; - - fn test_config(contents: Option<&str>, env_variables: Vec<(K, V)>, test_fn: F) - where - K: AsRef, - V: Display, - F: FnOnce(Config) { - Jail::expect_with(|jail| { - if let Some(contents) = contents { - jail.create_file("test.toml", contents)?; - } - - for (key, value) in env_variables { - jail.set_env(key, value); - } - - test_fn(Config::from_env("test.toml".into()).map_err(|err| err.to_string())?); - - Ok(()) - }); - } - - fn test_config_from_env(env_variables: Vec<(K, V)>, test_fn: F) - where - K: AsRef, - V: Display, - F: FnOnce(Config) { - test_config(None, env_variables, test_fn); - } - - fn test_config_from_file(contents: &str, test_fn: F) - where - F: FnOnce(Config) { - test_config(Some(contents), Vec::<(&str, &str)>::new(), test_fn); - } - - #[test] - fn config_ticket_server_addr_env() { - test_config_from_env(vec![("HTSGET_TICKET_SERVER_ADDR", "127.0.0.1:8082")], |config| { - assert_eq!( - config.ticket_server().addr(), - "127.0.0.1:8082".parse().unwrap() - ); - }); - } - - #[test] - fn config_ticket_server_cors_allow_origin_env() { - test_config_from_env(vec![("HTSGET_TICKET_SERVER_ALLOW_CREDENTIALS", true)], |config| { - assert!( - config.ticket_server().allow_credentials() - ); - }); - } - - #[test] - fn config_service_info_id_env() { - test_config_from_env(vec![("HTSGET_ID", "id")], |config| { - assert_eq!( - config.ticket_server().id(), - &Some("id".to_string()) - ); - }); - } - - #[test] - fn config_data_server_addr_env() { - test_config_from_env( vec![("HTSGET_DATA_SERVERS", "[{addr=127.0.0.1:8082}]")], |config| { - assert_eq!( - config.data_servers().first().unwrap().addr(), - "127.0.0.1:8082".parse().unwrap() - ); - }); - } - - #[test] - fn config_resolvers_env() { - test_config_from_env(vec![("HTSGET_RESOLVERS", "[{regex=regex}]")], |config| { - assert_eq!( - config.resolvers().first().unwrap().regex().as_str(), - "regex" - ); - }); - } - - #[test] - fn config_ticket_server_addr_file() { - test_config_from_file(r#"ticket_server_addr = "127.0.0.1:8082""#, |config| { - assert_eq!( - config.ticket_server().addr(), - "127.0.0.1:8082".parse().unwrap() - ); - }); - } - - #[test] - fn config_ticket_server_cors_allow_origin_file() { - test_config_from_file(r#"ticket_server_allow_credentials = true"#, |config| { - assert!( - config.ticket_server().allow_credentials() - ); - }); - } - - #[test] - fn config_service_info_id_file() { - test_config_from_file(r#"id = "id""#, |config| { - assert_eq!( - config.ticket_server().id(), - &Some("id".to_string()) - ); - }); - } - - #[test] - fn config_data_server_addr_file() { - test_config_from_file(r#" + use super::*; + use crate::regex_resolver::{Scheme, StorageType}; + use crate::Format::Bam; + use figment::Jail; + use std::fmt::Display; + + fn test_config(contents: Option<&str>, env_variables: Vec<(K, V)>, test_fn: F) + where + K: AsRef, + V: Display, + F: FnOnce(Config), + { + Jail::expect_with(|jail| { + if let Some(contents) = contents { + jail.create_file("test.toml", contents)?; + } + + for (key, value) in env_variables { + jail.set_env(key, value); + } + + test_fn(Config::from_env("test.toml".into()).map_err(|err| err.to_string())?); + + Ok(()) + }); + } + + fn test_config_from_env(env_variables: Vec<(K, V)>, test_fn: F) + where + K: AsRef, + V: Display, + F: FnOnce(Config), + { + test_config(None, env_variables, test_fn); + } + + fn test_config_from_file(contents: &str, test_fn: F) + where + F: FnOnce(Config), + { + test_config(Some(contents), Vec::<(&str, &str)>::new(), test_fn); + } + + #[test] + fn config_ticket_server_addr_env() { + test_config_from_env( + vec![("HTSGET_TICKET_SERVER_ADDR", "127.0.0.1:8082")], + |config| { + assert_eq!( + config.ticket_server().addr(), + "127.0.0.1:8082".parse().unwrap() + ); + }, + ); + } + + #[test] + fn config_ticket_server_cors_allow_origin_env() { + test_config_from_env( + vec![("HTSGET_TICKET_SERVER_ALLOW_CREDENTIALS", true)], + |config| { + assert!(config.ticket_server().allow_credentials()); + }, + ); + } + + #[test] + fn config_service_info_id_env() { + test_config_from_env(vec![("HTSGET_ID", "id")], |config| { + assert_eq!(config.ticket_server().id(), &Some("id".to_string())); + }); + } + + #[test] + fn config_data_server_addr_env() { + test_config_from_env( + vec![("HTSGET_DATA_SERVERS", "[{addr=127.0.0.1:8082}]")], + |config| { + assert_eq!( + config.data_server().unwrap().addr(), + "127.0.0.1:8082".parse().unwrap() + ); + }, + ); + } + + #[test] + fn config_resolvers_env() { + test_config_from_env(vec![("HTSGET_RESOLVERS", "[{regex=regex}]")], |config| { + assert_eq!( + config.resolvers().first().unwrap().regex().as_str(), + "regex" + ); + }); + } + + #[test] + fn config_ticket_server_addr_file() { + test_config_from_file(r#"ticket_server_addr = "127.0.0.1:8082""#, |config| { + assert_eq!( + config.ticket_server().addr(), + "127.0.0.1:8082".parse().unwrap() + ); + }); + } + + #[test] + fn config_ticket_server_cors_allow_origin_file() { + test_config_from_file(r#"ticket_server_allow_credentials = true"#, |config| { + assert!(config.ticket_server().allow_credentials()); + }); + } + + #[test] + fn config_service_info_id_file() { + test_config_from_file(r#"id = "id""#, |config| { + assert_eq!(config.ticket_server().id(), &Some("id".to_string())); + }); + } + + #[test] + fn config_data_server_addr_file() { + test_config_from_file( + r#" [[data_servers]] addr = "127.0.0.1:8082" - "#, |config| { - assert_eq!( - config.data_servers().first().unwrap().addr(), - "127.0.0.1:8082".parse().unwrap() - ); - }); - } - - #[test] - fn config_resolvers_file() { - test_config_from_file(r#" + "#, + |config| { + assert_eq!( + config.data_server().unwrap().addr(), + "127.0.0.1:8082".parse().unwrap() + ); + }, + ); + } + + #[test] + fn config_resolvers_file() { + test_config_from_file( + r#" [[resolvers]] regex = "regex" - "#, |config| { - assert_eq!( - config.resolvers().first().unwrap().regex().as_str(), - "regex" - ); - }); - } - - #[test] - fn config_resolvers_guard_file() { - test_config_from_file(r#" + "#, + |config| { + assert_eq!( + config.resolvers().first().unwrap().regex().as_str(), + "regex" + ); + }, + ); + } + + #[test] + fn config_resolvers_guard_file() { + test_config_from_file( + r#" [[resolvers]] regex = "regex" [resolvers.guard] match_formats = ["BAM"] - "#, |config| { - assert_eq!( - config.resolvers().first().unwrap().match_formats(), - &vec![Bam] - ); - }); - } - - #[test] - fn config_storage_type_url_file() { - test_config_from_file(r#" + "#, + |config| { + assert_eq!( + config.resolvers().first().unwrap().match_formats(), + &vec![Bam] + ); + }, + ); + } + + #[test] + fn config_storage_type_url_file() { + test_config_from_file( + r#" [[resolvers]] regex = "regex" @@ -572,29 +598,34 @@ mod tests { type = "Url" path = "path" scheme = "HTTPS" - "#, |config| { - assert!(matches!( - config.resolvers().first().unwrap().storage_type(), - StorageType::Url(resolver) if resolver.path() == "path" && resolver.scheme() == Scheme::Https - )); - }); - } - - #[cfg(feature = "s3-storage")] - #[test] - fn config_storage_type_s3_file() { - test_config_from_file(r#" + "#, + |config| { + assert!(matches!( + config.resolvers().first().unwrap().storage_type(), + StorageType::Url(resolver) if resolver.path() == "path" && resolver.scheme() == Scheme::Https + )); + }, + ); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn config_storage_type_s3_file() { + test_config_from_file( + r#" [[resolvers]] regex = "regex" [resolvers.storage_type] type = "S3" bucket = "bucket" - "#, |config| { - assert!(matches!( - config.resolvers().first().unwrap().storage_type(), - StorageType::S3(resolver) if resolver.bucket() == "bucket" - )); - }); - } + "#, + |config| { + assert!(matches!( + config.resolvers().first().unwrap().storage_type(), + StorageType::S3(resolver) if resolver.bucket() == "bucket" + )); + }, + ); + } } diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 0df656d61..1a54f04b5 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -1,28 +1,26 @@ use http::uri::Authority; use regex::{Error, Regex}; use serde::{Deserialize, Serialize}; -use serde_with::with_prefix; use tracing::instrument; use crate::config::{default_localstorage_addr, default_serve_at}; +use crate::regex_resolver::aws::S3Resolver; use crate::Format::{Bam, Bcf, Cram, Vcf}; use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; -use crate::regex_resolver::aws::S3Resolver; -use crate::regex_resolver::Scheme::Https; #[cfg(feature = "s3-storage")] pub mod aws; /// Represents an id resolver, which matches the id, replacing the match in the substitution text. -pub trait IdResolver { - /// Resolve the id, returning the substituted string if there is a match. - fn resolve_id(&self, query: &Query) -> Option; +pub trait Resolver { + /// Resolve the id, returning the substituted string if there is a match. + fn resolve_id(&mut self, query: &Query) -> Option; } /// Determines whether the query matches for use with the resolver. pub trait QueryMatcher { - /// Does this query match. - fn query_matches(&self, query: &Query) -> bool; + /// Does this query match. + fn query_matches(&self, query: &Query) -> bool; } /// Specify the storage type to use. @@ -30,297 +28,311 @@ pub trait QueryMatcher { #[serde(tag = "type")] #[non_exhaustive] pub enum StorageType { - #[serde(alias = "url", alias = "URL")] - Url(UrlResolver), - #[cfg(feature = "s3-storage")] - #[serde(alias = "s3")] - S3(S3Resolver), + #[serde(alias = "url", alias = "URL")] + Url(UrlResolver), + #[cfg(feature = "s3-storage")] + #[serde(alias = "s3")] + S3(S3Resolver), } impl Default for StorageType { - fn default() -> Self { - Self::Url(UrlResolver::default()) - } + fn default() -> Self { + Self::Url(UrlResolver::default()) + } } #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] pub enum Scheme { - #[serde(alias = "http", alias = "HTTP")] - Http, - #[serde(alias = "https", alias = "HTTPS")] - Https + #[serde(alias = "http", alias = "HTTP")] + Http, + #[serde(alias = "https", alias = "HTTPS")] + Https, } impl Default for Scheme { - fn default() -> Self { - Self::Http - } + fn default() -> Self { + Self::Http + } } /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct UrlResolver { - scheme: Scheme, - #[serde(with = "http_serde::authority")] - authority: Authority, - path: String, + scheme: Scheme, + #[serde(with = "http_serde::authority")] + authority: Authority, + path: String, } impl UrlResolver { - pub fn scheme(&self) -> Scheme { - self.scheme - } + pub fn scheme(&self) -> Scheme { + self.scheme + } - pub fn authority(&self) -> &Authority { - &self.authority - } + pub fn authority(&self) -> &Authority { + &self.authority + } - pub fn path(&self) -> &str { - &self.path - } + pub fn path(&self) -> &str { + &self.path + } } impl Default for UrlResolver { - fn default() -> Self { - Self { - scheme: Scheme::default(), - authority: Authority::from_static(default_localstorage_addr()), - path: default_serve_at().to_string() - } + fn default() -> Self { + Self { + scheme: Scheme::default(), + authority: Authority::from_static(default_localstorage_addr()), + path: default_serve_at().to_string(), } + } } /// A regex resolver is a resolver that matches ids using Regex. #[derive(Serialize, Debug, Clone, Deserialize)] #[serde(default)] pub struct RegexResolver { - #[serde(with = "serde_regex")] - regex: Regex, - // todo: should match guard be allowed as variables inside the substitution string? - substitution_string: String, - guard: QueryGuard, - storage_type: StorageType, + #[serde(with = "serde_regex")] + regex: Regex, + // todo: should match guard be allowed as variables inside the substitution string? + substitution_string: String, + guard: QueryGuard, + storage_type: StorageType, } /// A query that can be matched with the regex resolver. #[derive(Serialize, Clone, Debug, Deserialize)] #[serde(default)] pub struct QueryGuard { - match_formats: Vec, - match_class: Vec, - #[serde(with = "serde_regex")] - match_reference_name: Regex, - /// The start and end positions are 0-based. [start, end) - start_interval: Interval, - end_interval: Interval, - match_fields: Fields, - match_tags: Tags, - match_no_tags: NoTags, + match_formats: Vec, + match_class: Vec, + #[serde(with = "serde_regex")] + match_reference_name: Regex, + /// The start and end positions are 0-based. [start, end) + start_interval: Interval, + end_interval: Interval, + match_fields: Fields, + match_tags: Tags, + match_no_tags: NoTags, } impl QueryGuard { - pub fn match_formats(&self) -> &Vec { - &self.match_formats - } + pub fn match_formats(&self) -> &Vec { + &self.match_formats + } - pub fn match_class(&self) -> &Vec { - &self.match_class - } + pub fn match_class(&self) -> &Vec { + &self.match_class + } - pub fn match_reference_name(&self) -> &Regex { - &self.match_reference_name - } + pub fn match_reference_name(&self) -> &Regex { + &self.match_reference_name + } - pub fn start_interval(&self) -> Interval { - self.start_interval - } + pub fn start_interval(&self) -> Interval { + self.start_interval + } - pub fn end_interval(&self) -> Interval { - self.end_interval - } + pub fn end_interval(&self) -> Interval { + self.end_interval + } - pub fn match_fields(&self) -> &Fields { - &self.match_fields - } + pub fn match_fields(&self) -> &Fields { + &self.match_fields + } - pub fn match_tags(&self) -> &Tags { - &self.match_tags - } + pub fn match_tags(&self) -> &Tags { + &self.match_tags + } - pub fn match_no_tags(&self) -> &NoTags { - &self.match_no_tags - } + pub fn match_no_tags(&self) -> &NoTags { + &self.match_no_tags + } } impl Default for QueryGuard { - fn default() -> Self { - Self { - match_formats: vec![Bam, Cram, Vcf, Bcf], - match_class: vec![Class::Body, Class::Header], - match_reference_name: Regex::new(".*").expect("Expected valid regex expression"), - start_interval: Interval { start: Some(0), end: Some(100) }, - end_interval: Default::default(), - match_fields: Fields::All, - match_tags: Tags::All, - match_no_tags: NoTags(None), - } - } + fn default() -> Self { + Self { + match_formats: vec![Bam, Cram, Vcf, Bcf], + match_class: vec![Class::Body, Class::Header], + match_reference_name: Regex::new(".*").expect("Expected valid regex expression"), + start_interval: Interval { + start: Some(0), + end: Some(100), + }, + end_interval: Default::default(), + match_fields: Fields::All, + match_tags: Tags::All, + match_no_tags: NoTags(None), + } + } } impl QueryMatcher for Fields { - fn query_matches(&self, query: &Query) -> bool { - match (self, &query.fields) { - (Fields::All, _) => true, - (Fields::List(self_fields), Fields::List(query_fields)) => self_fields == query_fields, - (Fields::List(_), Fields::All) => false, - } + fn query_matches(&self, query: &Query) -> bool { + match (self, &query.fields) { + (Fields::All, _) => true, + (Fields::List(self_fields), Fields::List(query_fields)) => self_fields == query_fields, + (Fields::List(_), Fields::All) => false, } + } } impl QueryMatcher for Tags { - fn query_matches(&self, query: &Query) -> bool { - match (self, &query.tags) { - (Tags::All, _) => true, - (Tags::List(self_tags), Tags::List(query_tags)) => self_tags == query_tags, - (Tags::List(_), Tags::All) => false, - } + fn query_matches(&self, query: &Query) -> bool { + match (self, &query.tags) { + (Tags::All, _) => true, + (Tags::List(self_tags), Tags::List(query_tags)) => self_tags == query_tags, + (Tags::List(_), Tags::All) => false, } + } } impl QueryMatcher for NoTags { - fn query_matches(&self, query: &Query) -> bool { - match (self, &query.no_tags) { - (NoTags(None), _) => true, - (NoTags(Some(self_no_tags)), NoTags(Some(query_no_tags))) => self_no_tags == query_no_tags, - (NoTags(Some(_)), NoTags(None)) => false, - } + fn query_matches(&self, query: &Query) -> bool { + match (self, &query.no_tags) { + (NoTags(None), _) => true, + (NoTags(Some(self_no_tags)), NoTags(Some(query_no_tags))) => self_no_tags == query_no_tags, + (NoTags(Some(_)), NoTags(None)) => false, } + } } impl QueryMatcher for QueryGuard { - fn query_matches(&self, query: &Query) -> bool { - if let Some(reference_name) = &query.reference_name { - self.match_formats.contains(&query.format) - && self.match_class.contains(&query.class) - && self.match_reference_name.is_match(reference_name) - && self - .start_interval - .contains(query.interval.start.unwrap_or(u32::MIN)) - && self.end_interval.contains(query.interval.end.unwrap_or(u32::MAX)) - && self.match_fields.query_matches(query) - && self.match_tags.query_matches(query) - && self.match_no_tags.query_matches(query) - } else { - false - } - } + fn query_matches(&self, query: &Query) -> bool { + if let Some(reference_name) = &query.reference_name { + self.match_formats.contains(&query.format) + && self.match_class.contains(&query.class) + && self.match_reference_name.is_match(reference_name) + && self + .start_interval + .contains(query.interval.start.unwrap_or(u32::MIN)) + && self + .end_interval + .contains(query.interval.end.unwrap_or(u32::MAX)) + && self.match_fields.query_matches(query) + && self.match_tags.query_matches(query) + && self.match_no_tags.query_matches(query) + } else { + false + } + } } impl Default for RegexResolver { - fn default() -> Self { - Self::new(StorageType::default(), ".*", "$0", QueryGuard::default()) - .expect("expected valid resolver") - } + fn default() -> Self { + Self::new(StorageType::default(), ".*", "$0", QueryGuard::default()) + .expect("expected valid resolver") + } } impl RegexResolver { - /// Create a new regex resolver. - pub fn new( - storage_type: StorageType, - regex: &str, - replacement_string: &str, - guard: QueryGuard, - ) -> Result { - Ok(Self { - regex: Regex::new(regex)?, - substitution_string: replacement_string.to_string(), - storage_type, - guard, - }) - } - - pub fn regex(&self) -> &Regex { - &self.regex - } - - pub fn substitution_string(&self) -> &str { - &self.substitution_string - } - - pub fn guard(&self) -> &QueryGuard { - &self.guard - } - - pub fn storage_type(&self) -> &StorageType { - &self.storage_type - } - - pub fn match_formats(&self) -> &Vec { - &self.guard.match_formats - } - - pub fn match_class(&self) -> &Vec { - &self.guard.match_class - } - - pub fn match_reference_name(&self) -> &Regex { - &self.guard.match_reference_name - } - - pub fn start_interval(&self) -> Interval { - self.guard.start_interval - } - - pub fn end_interval(&self) -> Interval { - self.guard.end_interval - } - - pub fn match_fields(&self) -> &Fields { - &self.guard.match_fields - } - - pub fn match_tags(&self) -> &Tags { - &self.guard.match_tags - } + /// Create a new regex resolver. + pub fn new( + storage_type: StorageType, + regex: &str, + replacement_string: &str, + guard: QueryGuard, + ) -> Result { + Ok(Self { + regex: Regex::new(regex)?, + substitution_string: replacement_string.to_string(), + storage_type, + guard, + }) + } + + pub fn regex(&self) -> &Regex { + &self.regex + } + + pub fn substitution_string(&self) -> &str { + &self.substitution_string + } + + pub fn guard(&self) -> &QueryGuard { + &self.guard + } + + pub fn storage_type(&self) -> &StorageType { + &self.storage_type + } + + pub fn match_formats(&self) -> &Vec { + &self.guard.match_formats + } + + pub fn match_class(&self) -> &Vec { + &self.guard.match_class + } + + pub fn match_reference_name(&self) -> &Regex { + &self.guard.match_reference_name + } + + pub fn start_interval(&self) -> Interval { + self.guard.start_interval + } + + pub fn end_interval(&self) -> Interval { + self.guard.end_interval + } + + pub fn match_fields(&self) -> &Fields { + &self.guard.match_fields + } + + pub fn match_tags(&self) -> &Tags { + &self.guard.match_tags + } + + pub fn match_no_tags(&self) -> &NoTags { + &self.guard.match_no_tags + } +} - pub fn match_no_tags(&self) -> &NoTags { - &self.guard.match_no_tags - } +impl Resolver for RegexResolver { + #[instrument(level = "trace", skip(self), ret)] + fn resolve_id(&mut self, query: &Query) -> Option { + if self.regex.is_match(&query.id) && self.guard.query_matches(query) { + Some( + self + .regex + .replace(&query.id, &self.substitution_string) + .to_string(), + ) + } else { + None + } + } } -impl IdResolver for RegexResolver { - #[instrument(level = "trace", skip(self), ret)] - fn resolve_id(&self, query: &Query) -> Option { - if self.regex.is_match(&query.id) && self.guard.query_matches(query) { - Some( - self - .regex - .replace(&query.id, &self.substitution_string) - .to_string(), - ) - } else { - None - } - } +impl Resolver for I +where + I: Iterator, +{ + fn resolve_id(&mut self, query: &Query) -> Option { + self.find_map(|mut resolver| resolver.resolve_id(query)) + } } #[cfg(test)] pub mod tests { - use super::*; - - #[test] - fn resolver_resolve_id() { - let resolver = RegexResolver::new( - StorageType::default(), - ".*", - "$0-test", - QueryGuard::default(), - ) - .unwrap(); - assert_eq!( - resolver.resolve_id(&Query::new("id", Bam)).unwrap(), - "id-test" - ); - } + use super::*; + + #[test] + fn resolver_resolve_id() { + let mut resolver = RegexResolver::new( + StorageType::default(), + ".*", + "$0-test", + QueryGuard::default(), + ) + .unwrap(); + assert_eq!( + resolver.resolve_id(&Query::new("id", Bam)).unwrap(), + "id-test" + ); + } } diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index 0e7a5a67c..befb0087b 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -3,7 +3,7 @@ pub use htsget_config::config::aws::AwsS3DataServer; pub use htsget_config::config::{ Config, LocalDataServer, ServiceInfo, StorageType, TicketServerConfig, }; -pub use htsget_config::regex_resolver::{IdResolver, RegexResolver}; +pub use htsget_config::regex_resolver::{Resolver, RegexResolver}; pub mod htsget; pub mod storage; diff --git a/htsget-search/src/storage/mod.rs b/htsget-search/src/storage/mod.rs index b320e837d..d7fe968c8 100644 --- a/htsget-search/src/storage/mod.rs +++ b/htsget-search/src/storage/mod.rs @@ -19,7 +19,7 @@ use tracing::instrument; use crate::htsget::{Headers, Url}; use crate::storage::data_server::CORS_MAX_AGE; use crate::storage::StorageError::DataServerError; -use crate::{IdResolver, RegexResolver}; +use crate::{Resolver, RegexResolver}; #[cfg(feature = "s3-storage")] pub mod aws; From 2e2256d7dc1a92129a83a56790530c6d1e6f970a Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 19 Dec 2022 09:08:57 +1100 Subject: [PATCH 24/45] config: update getter return types --- htsget-config/src/config/mod.rs | 92 ++++++++++++------------- htsget-config/src/regex_resolver/mod.rs | 12 ++-- 2 files changed, 51 insertions(+), 53 deletions(-) diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 67b3076fb..d9670c439 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -4,7 +4,7 @@ use std::fmt::Debug; use std::io; use std::io::ErrorKind; use std::net::SocketAddr; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAnyAllowType}; use clap::Parser; @@ -12,9 +12,8 @@ use figment::providers::{Env, Format, Serialized, Toml}; use figment::Figment; use http::header::HeaderName; use http::Method; -use regex::internal::Input; -use serde::{de, Deserialize, Deserializer, Serialize}; use serde::de::IntoDeserializer; +use serde::{Deserialize, Deserializer, Serialize}; use serde_with::with_prefix; use tracing::info; use tracing::instrument; @@ -93,13 +92,12 @@ struct Args { fn empty_string_as_none<'de, D, T>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, - T: Deserialize<'de> + T: Deserialize<'de>, { - let optional_string = Option::deserialize(deserializer)?.filter(|s: &String| !s.is_empty() && s.to_lowercase() != "none"); + let optional_string = Option::deserialize(deserializer)? + .filter(|s: &String| !s.is_empty() && s.to_lowercase() != "none"); if let Some(string) = optional_string { - Ok(Some( - T::deserialize(string.into_deserializer())?, - )) + Ok(Some(T::deserialize(string.into_deserializer())?)) } else { Ok(None) } @@ -166,43 +164,43 @@ impl TicketServerConfig { self.cors.expose_headers() } - pub fn id(&self) -> &Option { + pub fn id(&self) -> Option<&str> { self.service_info.id() } - pub fn name(&self) -> &Option { + pub fn name(&self) -> Option<&str> { self.service_info.name() } - pub fn version(&self) -> &Option { + pub fn version(&self) -> Option<&str> { self.service_info.version() } - pub fn organization_name(&self) -> &Option { + pub fn organization_name(&self) -> Option<&str> { self.service_info.organization_name() } - pub fn organization_url(&self) -> &Option { + pub fn organization_url(&self) -> Option<&str> { self.service_info.organization_url() } - pub fn contact_url(&self) -> &Option { + pub fn contact_url(&self) -> Option<&str> { self.service_info.contact_url() } - pub fn documentation_url(&self) -> &Option { + pub fn documentation_url(&self) -> Option<&str> { self.service_info.documentation_url() } - pub fn created_at(&self) -> &Option { + pub fn created_at(&self) -> Option<&str> { self.service_info.created_at() } - pub fn updated_at(&self) -> &Option { + pub fn updated_at(&self) -> Option<&str> { self.service_info.updated_at() } - pub fn environment(&self) -> &Option { + pub fn environment(&self) -> Option<&str> { self.service_info.environment() } } @@ -225,20 +223,20 @@ impl DataServerConfig { self.addr } - pub fn path(&self) -> &PathBuf { + pub fn path(&self) -> &Path { &self.path } - pub fn serve_at(&self) -> &PathBuf { + pub fn serve_at(&self) -> &Path { &self.serve_at } - pub fn key(&self) -> &Option { - &self.key + pub fn key(&self) -> Option<&Path> { + self.key.as_deref() } - pub fn cert(&self) -> &Option { - &self.cert + pub fn cert(&self) -> Option<&Path> { + self.cert.as_deref() } pub fn cors(&self) -> &CorsConfig { @@ -302,44 +300,44 @@ pub struct ServiceInfo { } impl ServiceInfo { - pub fn id(&self) -> &Option { - &self.id + pub fn id(&self) -> Option<&str> { + self.id.as_deref() } - pub fn name(&self) -> &Option { - &self.name + pub fn name(&self) -> Option<&str> { + self.name.as_deref() } - pub fn version(&self) -> &Option { - &self.version + pub fn version(&self) -> Option<&str> { + self.version.as_deref() } - pub fn organization_name(&self) -> &Option { - &self.organization_name + pub fn organization_name(&self) -> Option<&str> { + self.organization_name.as_deref() } - pub fn organization_url(&self) -> &Option { - &self.organization_url + pub fn organization_url(&self) -> Option<&str> { + self.organization_url.as_deref() } - pub fn contact_url(&self) -> &Option { - &self.contact_url + pub fn contact_url(&self) -> Option<&str> { + self.contact_url.as_deref() } - pub fn documentation_url(&self) -> &Option { - &self.documentation_url + pub fn documentation_url(&self) -> Option<&str> { + self.documentation_url.as_deref() } - pub fn created_at(&self) -> &Option { - &self.created_at + pub fn created_at(&self) -> Option<&str> { + self.created_at.as_deref() } - pub fn updated_at(&self) -> &Option { - &self.updated_at + pub fn updated_at(&self) -> Option<&str> { + self.updated_at.as_deref() } - pub fn environment(&self) -> &Option { - &self.environment + pub fn environment(&self) -> Option<&str> { + self.environment.as_deref() } } @@ -409,7 +407,7 @@ impl Config { self.data_server.as_ref() } - pub fn resolvers(&self) -> &Vec { + pub fn resolvers(&self) -> &[RegexResolver] { &self.resolvers } } @@ -485,7 +483,7 @@ mod tests { #[test] fn config_service_info_id_env() { test_config_from_env(vec![("HTSGET_ID", "id")], |config| { - assert_eq!(config.ticket_server().id(), &Some("id".to_string())); + assert_eq!(config.ticket_server().id(), Some("id")); }); } @@ -532,7 +530,7 @@ mod tests { #[test] fn config_service_info_id_file() { test_config_from_file(r#"id = "id""#, |config| { - assert_eq!(config.ticket_server().id(), &Some("id".to_string())); + assert_eq!(config.ticket_server().id(), Some("id")); }); } diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 1a54f04b5..a52841f10 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -118,11 +118,11 @@ pub struct QueryGuard { } impl QueryGuard { - pub fn match_formats(&self) -> &Vec { + pub fn match_formats(&self) -> &[Format] { &self.match_formats } - pub fn match_class(&self) -> &Vec { + pub fn match_classes(&self) -> &[Class] { &self.match_class } @@ -259,12 +259,12 @@ impl RegexResolver { &self.storage_type } - pub fn match_formats(&self) -> &Vec { - &self.guard.match_formats + pub fn match_formats(&self) -> &[Format] { + self.guard.match_formats() } - pub fn match_class(&self) -> &Vec { - &self.guard.match_class + pub fn match_classes(&self) -> &[Class] { + self.guard.match_classes() } pub fn match_reference_name(&self) -> &Regex { From 945cc5774668702503844e1d4e6e57bfcb75e300 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 19 Dec 2022 22:39:11 +1100 Subject: [PATCH 25/45] refactor: apply changes to other crates from reworked config --- htsget-config/src/config/cors.rs | 30 ++++++ htsget-config/src/config/mod.rs | 40 ++++++++ htsget-config/src/lib.rs | 5 +- htsget-config/src/regex_resolver/mod.rs | 22 ++--- .../benches/request_benchmarks.rs | 4 +- htsget-http-actix/src/lib.rs | 95 +++++++++++-------- htsget-http-actix/src/main.rs | 51 ++++------ htsget-http-core/src/lib.rs | 21 ++-- htsget-http-core/src/query_builder.rs | 36 +++---- htsget-http-core/src/service_info.rs | 40 ++++---- htsget-http-lambda/src/lib.rs | 60 ++++-------- htsget-http-lambda/src/main.rs | 45 ++------- htsget-search/benches/search_benchmarks.rs | 15 +-- htsget-search/src/htsget/cram_search.rs | 4 +- htsget-search/src/htsget/from_storage.rs | 71 +++++++++----- htsget-search/src/htsget/mod.rs | 37 +++++--- htsget-search/src/htsget/search.rs | 28 +++--- htsget-search/src/lib.rs | 8 +- htsget-search/src/storage/aws.rs | 27 ++---- htsget-search/src/storage/data_server.rs | 81 ++++++---------- htsget-search/src/storage/local.rs | 28 +++--- htsget-search/src/storage/mod.rs | 93 ++++++++++++------ htsget-test-utils/src/http_tests.rs | 54 +++++++---- htsget-test-utils/src/lib.rs | 12 ++- htsget-test-utils/src/server_tests.rs | 16 +--- 25 files changed, 487 insertions(+), 436 deletions(-) diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs index b49777ddf..adfbca7aa 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/cors.rs @@ -68,6 +68,12 @@ where #[derive(Debug, Clone, PartialEq, Eq)] pub struct HeaderValue(HeaderValueInner); +impl HeaderValue { + pub fn into_inner(self) -> HeaderValueInner { + self.0 + } +} + impl FromStr for HeaderValue { type Err = InvalidHeaderValue; @@ -118,6 +124,30 @@ impl CorsConfig { pub fn expose_headers(&self) -> &AllowType { &self.expose_headers } + + pub fn set_allow_credentials(&mut self, allow_credentials: bool) { + self.allow_credentials = allow_credentials; + } + + pub fn set_allow_origins(&mut self, allow_origins: AllowType) { + self.allow_origins = allow_origins; + } + + pub fn set_allow_headers(&mut self, allow_headers: AllowType) { + self.allow_headers = allow_headers; + } + + pub fn set_allow_methods(&mut self, allow_methods: AllowType) { + self.allow_methods = allow_methods; + } + + pub fn set_max_age(&mut self, max_age: usize) { + self.max_age = max_age; + } + + pub fn set_expose_headers(&mut self, expose_headers: AllowType) { + self.expose_headers = expose_headers; + } } impl Default for CorsConfig { diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index d9670c439..f04c5aa73 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -266,6 +266,30 @@ impl DataServerConfig { pub fn expose_headers(&self) -> &AllowType { self.cors.expose_headers() } + + pub fn set_addr(&mut self, addr: SocketAddr) { + self.addr = addr; + } + + pub fn set_path(&mut self, path: PathBuf) { + self.path = path; + } + + pub fn set_serve_at(&mut self, serve_at: PathBuf) { + self.serve_at = serve_at; + } + + pub fn set_key(&mut self, key: Option) { + self.key = key; + } + + pub fn set_cert(&mut self, cert: Option) { + self.cert = cert; + } + + pub fn set_cors(&mut self, cors: CorsConfig) { + self.cors = cors; + } } impl Default for DataServerConfig { @@ -410,6 +434,22 @@ impl Config { pub fn resolvers(&self) -> &[RegexResolver] { &self.resolvers } + + pub fn owned_resolvers(self) -> Vec { + self.resolvers + } + + pub fn set_ticket_server(&mut self, ticket_server: TicketServerConfig) { + self.ticket_server = ticket_server; + } + + pub fn set_data_server(&mut self, data_server: Option) { + self.data_server = data_server; + } + + pub fn set_resolvers(&mut self, resolvers: Vec) { + self.resolvers = resolvers; + } } #[cfg(test)] diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 66a7cc3f7..69a61a45c 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -25,6 +25,7 @@ pub enum Format { Bcf, } +/// Todo allow these to be configurable. impl Format { pub fn fmt_file(&self, id: &str) -> String { match self { @@ -273,8 +274,8 @@ impl Query { self.class } - pub fn reference_name(&self) -> &Option { - &self.reference_name + pub fn reference_name(&self) -> Option<&str> { + self.reference_name.as_deref() } pub fn interval(&self) -> Interval { diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index a52841f10..248abad9e 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -14,7 +14,7 @@ pub mod aws; /// Represents an id resolver, which matches the id, replacing the match in the substitution text. pub trait Resolver { /// Resolve the id, returning the substituted string if there is a match. - fn resolve_id(&mut self, query: &Query) -> Option; + fn resolve_id(&self, query: &Query) -> Option; } /// Determines whether the query matches for use with the resolver. @@ -95,7 +95,7 @@ impl Default for UrlResolver { pub struct RegexResolver { #[serde(with = "serde_regex")] regex: Regex, - // todo: should match guard be allowed as variables inside the substitution string? + // Todo: should match guard be allowed as variables inside the substitution string? substitution_string: String, guard: QueryGuard, storage_type: StorageType, @@ -294,7 +294,7 @@ impl RegexResolver { impl Resolver for RegexResolver { #[instrument(level = "trace", skip(self), ret)] - fn resolve_id(&mut self, query: &Query) -> Option { + fn resolve_id(&self, query: &Query) -> Option { if self.regex.is_match(&query.id) && self.guard.query_matches(query) { Some( self @@ -308,14 +308,14 @@ impl Resolver for RegexResolver { } } -impl Resolver for I -where - I: Iterator, -{ - fn resolve_id(&mut self, query: &Query) -> Option { - self.find_map(|mut resolver| resolver.resolve_id(query)) - } -} +// impl<'a, I> Resolver for I +// where +// I: Iterator, +// { +// fn resolve_id(&self, query: &Query) -> Option { +// self.find_map(|resolver| resolver.resolve_id(query)) +// } +// } #[cfg(test)] pub mod tests { diff --git a/htsget-http-actix/benches/request_benchmarks.rs b/htsget-http-actix/benches/request_benchmarks.rs index da3e15aee..c9075dbd1 100644 --- a/htsget-http-actix/benches/request_benchmarks.rs +++ b/htsget-http-actix/benches/request_benchmarks.rs @@ -149,9 +149,9 @@ fn start_htsget_rs() -> (DropGuard, String) { .spawn() .unwrap(); - let htsget_rs_url = format!("http://{}", config.ticket_server_config.ticket_server_addr); + let htsget_rs_url = format!("http://{}", config.ticket_server().addr()); query_server_until_response(&format_url(&htsget_rs_url, "reads/service-info")); - let htsget_rs_ticket_url = format!("http://{}", config.data_server_config.data_server_addr); + let htsget_rs_ticket_url = format!("http://{}", config.data_server().unwrap().addr()); query_server_until_response(&format_url(&htsget_rs_ticket_url, "")); (DropGuard(child), htsget_rs_url) diff --git a/htsget-http-actix/src/lib.rs b/htsget-http-actix/src/lib.rs index 574a68c03..8c9b20c0b 100644 --- a/htsget-http-actix/src/lib.rs +++ b/htsget-http-actix/src/lib.rs @@ -7,11 +7,11 @@ use tracing::info; use tracing::instrument; use tracing_actix_web::TracingLogger; +use htsget_config::config::cors::{AllowType, CorsConfig, TaggedAllowTypes}; +pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig, USAGE}; #[cfg(feature = "s3-storage")] -pub use htsget_config::config::aws::AwsS3DataServer; -pub use htsget_config::config::{ - Config, LocalDataServer, ServiceInfo, StorageType, TicketServerConfig, USAGE, -}; +pub use htsget_config::regex_resolver::aws::S3Resolver; +pub use htsget_config::regex_resolver::StorageType; use htsget_search::htsget::from_storage::HtsGetFromStorage; use htsget_search::htsget::HtsGet; use htsget_search::storage::local::LocalStorage; @@ -60,18 +60,47 @@ pub fn configure_server( /// Configure cors, settings allowed methods, max age, allowed origins, and if credentials /// are supported. -pub fn configure_cors(cors_allow_credentials: bool, cors_allow_origin: String) -> Cors { - let cors = Cors::default() - .allow_any_method() - .allow_any_header() - .allowed_origin(&cors_allow_origin) - .max_age(CORS_MAX_AGE); - - if cors_allow_credentials { - cors.supports_credentials() - } else { - cors +pub fn configure_cors(cors: CorsConfig) -> Cors { + let mut cors_layer = Cors::default(); + cors_layer = match cors.allow_origins() { + AllowType::Tagged(tagged) => match tagged { + TaggedAllowTypes::Mirror => cors_layer.allow_any_origin(), + TaggedAllowTypes::Any => cors_layer.allow_any_origin().send_wildcard(), + }, + AllowType::List(origins) => { + for origin in origins { + cors_layer = cors_layer.allowed_origin(&origin.to_string()); + } + cors_layer + } + }; + + cors_layer = match cors.allow_headers() { + AllowType::Tagged(tagged) => match tagged { + TaggedAllowTypes::Mirror => cors_layer.allow_any_header(), + TaggedAllowTypes::Any => cors_layer.allow_any_header(), + }, + AllowType::List(headers) => cors_layer.allowed_headers(headers.clone()), + }; + + cors_layer = match cors.allow_methods() { + AllowType::Tagged(tagged) => match tagged { + TaggedAllowTypes::Mirror => cors_layer.allow_any_method(), + TaggedAllowTypes::Any => cors_layer.allow_any_method(), + }, + AllowType::List(methods) => cors_layer.allowed_methods(methods.clone()), + }; + + cors_layer = match cors.expose_headers() { + AllowType::Tagged(_) => cors_layer.expose_any_header(), + AllowType::List(headers) => cors_layer.expose_headers(headers.clone()), + }; + + if cors.allow_credentials() { + cors_layer = cors_layer.supports_credentials(); } + + cors_layer.max_age(cors.max_age()) } /// Run the server using a http-actix `HttpServer`. @@ -80,18 +109,21 @@ pub fn run_server( htsget: H, config: TicketServerConfig, ) -> std::io::Result { + let addr = config.addr(); + let server = HttpServer::new(Box::new(move || { App::new() .configure(|service_config: &mut web::ServiceConfig| { - configure_server(service_config, htsget.clone(), config.service_info.clone()); + configure_server( + service_config, + htsget.clone(), + config.service_info().clone(), + ); }) - .wrap(configure_cors( - config.cors_allow_credentials, - config.cors_allow_origin.clone(), - )) + .wrap(configure_cors(config.cors().clone())) .wrap(TracingLogger::default()) })) - .bind(config.addr)?; + .bind(addr)?; info!(addresses = ?server.addrs(), "htsget query server addresses bound"); Ok(server.run()) @@ -202,26 +234,11 @@ mod tests { .configure(|service_config: &mut web::ServiceConfig| { configure_server( service_config, - HtsGetFromStorage::local_from( - self.config.path.clone(), - self.config.resolver.clone(), - formatter, - ) - .unwrap(), - self.config.ticket_server_config.service_info.clone(), + self.config.clone().owned_resolvers(), + self.config.ticket_server().service_info().clone(), ); }) - .wrap(configure_cors( - self - .config - .data_server_config - .data_server_cors_allow_credentials, - self - .config - .data_server_config - .data_server_cors_allow_origin - .clone(), - )), + .wrap(configure_cors(self.config.ticket_server().cors().clone())), ) .await; diff --git a/htsget-http-actix/src/main.rs b/htsget-http-actix/src/main.rs index 4aa55881a..7d2c3b996 100644 --- a/htsget-http-actix/src/main.rs +++ b/htsget-http-actix/src/main.rs @@ -1,9 +1,8 @@ use std::io::{Error, ErrorKind}; -use tokio::select; -use htsget_config::config::aws::AwsS3DataServer; -use htsget_config::config::{LocalDataServer, TicketServerConfig}; +use htsget_config::config::{DataServerConfig, TicketServerConfig}; use htsget_config::regex_resolver::RegexResolver; +use tokio::select; use htsget_http_actix::run_server; use htsget_http_actix::{Config, StorageType}; @@ -15,34 +14,22 @@ async fn main() -> std::io::Result<()> { Config::setup_tracing()?; let config = Config::from_env(Config::parse_args())?; - let resolver = config.resolvers.first().unwrap(); - match resolver.server.clone() { - StorageType::LocalStorage(server_config) => local_storage_server(server_config.clone(), resolver, config.ticket_server_config).await, - #[cfg(feature = "s3-storage")] - StorageType::AwsS3Storage(server_config) => s3_storage_server(&server_config, resolver, config.ticket_server_config).await, - _ => Err(Error::new(ErrorKind::Other, "unsupported storage type")), + if let Some(server) = config.data_server() { + let server = server.clone(); + let mut formatter = HttpTicketFormatter::try_from(server.clone())?; + let local_server = formatter.bind_data_server().await?; + let local_server = tokio::spawn(async move { local_server.serve(&server.path()).await }); + + let ticket_server_config = config.ticket_server().clone(); + select! { + local_server = local_server => Ok(local_server??), + actix_server = run_server( + config.owned_resolvers(), + ticket_server_config, + )? => actix_server + } + } else { + let ticket_server_config = config.ticket_server().clone(); + run_server(config.owned_resolvers(), ticket_server_config)?.await } } - -async fn local_storage_server(config: LocalDataServer, resolver: &RegexResolver, ticket_config: TicketServerConfig) -> std::io::Result<()> { - let mut formatter = HttpTicketFormatter::try_from(config.clone())?; - let local_server = formatter.bind_data_server().await?; - - let searcher = - HtsGetFromStorage::local_from(config.path.clone(), resolver.clone(), formatter)?; - let local_server = tokio::spawn(async move { local_server.serve(&config.path.clone()).await }); - - select! { - local_server = local_server => Ok(local_server??), - actix_server = run_server( - searcher, - ticket_config, - )? => actix_server - } -} - -#[cfg(feature = "s3-storage")] -async fn s3_storage_server(config: &AwsS3DataServer, resolver: &RegexResolver, ticket_config: TicketServerConfig) -> std::io::Result<()> { - let searcher = HtsGetFromStorage::s3_from(config.bucket.clone(), resolver.clone()).await; - run_server(searcher, ticket_config)?.await -} diff --git a/htsget-http-core/src/lib.rs b/htsget-http-core/src/lib.rs index 7106fa793..e13bf4a31 100644 --- a/htsget-http-core/src/lib.rs +++ b/htsget-http-core/src/lib.rs @@ -2,11 +2,12 @@ use std::collections::HashMap; use std::str::FromStr; pub use error::{HtsGetError, Result}; -#[cfg(feature = "s3-storage")] -pub use htsget_config::config::aws::AwsS3DataServer; pub use htsget_config::config::{ - Config, LocalDataServer, ServiceInfo as ConfigServiceInfo, StorageType, TicketServerConfig, + Config, DataServerConfig, ServiceInfo as ConfigServiceInfo, TicketServerConfig, }; +#[cfg(feature = "s3-storage")] +pub use htsget_config::regex_resolver::aws::S3Resolver; +pub use htsget_config::regex_resolver::StorageType; use htsget_config::Query; use htsget_search::htsget::Response; pub use http_core::{get_response_for_get_request, get_response_for_post_request}; @@ -113,11 +114,11 @@ fn merge_responses(responses: Vec) -> Option { #[cfg(test)] mod tests { + use htsget_config::config::cors::CorsConfig; use std::path::PathBuf; use std::sync::Arc; - use htsget_config::config::StorageTypeServer; - use htsget_config::regex_resolver::{MatchOnQuery, RegexResolver}; + use htsget_config::regex_resolver::{RegexResolver, StorageType}; use htsget_config::Format; use htsget_search::htsget::HtsGet; use htsget_search::storage::data_server::HttpTicketFormatter; @@ -275,14 +276,8 @@ mod tests { Arc::new(HtsGetFromStorage::new( LocalStorage::new( get_base_path(), - RegexResolver::new( - ".*", - "$0", - StorageTypeServer::default(), - MatchOnQuery::default(), - ) - .unwrap(), - HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), "".to_string(), false), + RegexResolver::new(Default::default(), ".*", "$0", Default::default()).unwrap(), + HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), CorsConfig::default()), ) .unwrap(), )) diff --git a/htsget-http-core/src/query_builder.rs b/htsget-http-core/src/query_builder.rs index c25ec5923..cc8c6b286 100644 --- a/htsget-http-core/src/query_builder.rs +++ b/htsget-http-core/src/query_builder.rs @@ -99,11 +99,10 @@ impl QueryBuilder { self.query = self.query.with_end(end); } - if (self.query.interval.start.is_some() || self.query.interval.end.is_some()) + if (self.query.interval().start().is_some() || self.query.interval().end().is_some()) && self .query - .reference_name - .as_ref() + .reference_name() .filter(|name| *name != "*") .is_none() { @@ -112,7 +111,8 @@ impl QueryBuilder { )); } - if let (Some(start), Some(end)) = &(self.query.interval.start, self.query.interval.end) { + if let (Some(start), Some(end)) = &(self.query.interval().start(), self.query.interval().end()) + { if start > end { return Err(HtsGetError::InvalidRange(format!( "end is greater than start (`{}` > `{}`)", @@ -201,7 +201,7 @@ mod tests { QueryBuilder::new(Some("ValidId".to_string()), Some("BAM")) .unwrap() .build() - .id, + .id(), "ValidId".to_string() ); } @@ -212,7 +212,7 @@ mod tests { QueryBuilder::new(Some("ValidID"), Some("VCF")) .unwrap() .build() - .format, + .format(), Format::Vcf ); } @@ -233,7 +233,7 @@ mod tests { .with_class(Some("header")) .unwrap() .build() - .class, + .class(), Class::Header ); } @@ -245,8 +245,8 @@ mod tests { .unwrap() .with_reference_name(Some("ValidName")) .build() - .reference_name, - Some("ValidName".to_string()) + .reference_name(), + Some("ValidName") ); } @@ -259,7 +259,7 @@ mod tests { .unwrap() .build(); assert_eq!( - (query.interval.start, query.interval.end), + (query.interval().start(), query.interval().end()), (Some(3), Some(5)) ); } @@ -318,8 +318,8 @@ mod tests { .unwrap() .with_fields(Some("header,part1,part2")) .build() - .fields, - Fields::List(vec![ + .fields(), + &Fields::List(vec![ "header".to_string(), "part1".to_string(), "part2".to_string() @@ -335,14 +335,14 @@ mod tests { .unwrap() .build(); assert_eq!( - query.tags, - Tags::List(vec![ + query.tags(), + &Tags::List(vec![ "header".to_string(), "part1".to_string(), "part2".to_string() ]) ); - assert_eq!(query.no_tags, NoTags(Some(vec!["part3".to_string()]))); + assert_eq!(query.no_tags(), &NoTags(Some(vec!["part3".to_string()]))); } #[test] @@ -353,13 +353,13 @@ mod tests { .unwrap() .build(); assert_eq!( - query.tags, - Tags::List(vec![ + query.tags(), + &Tags::List(vec![ "header".to_string(), "part1".to_string(), "part2".to_string() ]) ); - assert_eq!(query.no_tags, NoTags(Some(vec!["part3".to_string()]))); + assert_eq!(query.no_tags(), &NoTags(Some(vec!["part3".to_string()]))); } } diff --git a/htsget-http-core/src/service_info.rs b/htsget-http-core/src/service_info.rs index e102cfbc3..a8b6f936c 100644 --- a/htsget-http-core/src/service_info.rs +++ b/htsget-http-core/src/service_info.rs @@ -118,35 +118,35 @@ fn fill_out_service_info_json( mut service_info_json: ServiceInfo, config: &ConfigServiceInfo, ) -> ServiceInfo { - if let Some(id) = &config.id { - service_info_json.id = id.clone(); + if let Some(id) = config.id() { + service_info_json.id = id.to_string(); } - if let Some(name) = &config.name { - service_info_json.name = name.clone(); + if let Some(name) = config.name() { + service_info_json.name = name.to_string(); } - if let Some(version) = &config.version { - service_info_json.version = version.clone(); + if let Some(version) = config.version() { + service_info_json.version = version.to_string(); } - if let Some(organization_name) = &config.organization_name { - service_info_json.organization.name = organization_name.clone(); + if let Some(organization_name) = config.organization_name() { + service_info_json.organization.name = organization_name.to_string(); } - if let Some(organization_url) = &config.organization_url { - service_info_json.organization.url = organization_url.clone(); + if let Some(organization_url) = config.organization_url() { + service_info_json.organization.url = organization_url.to_string(); } - if let Some(contact_url) = &config.contact_url { - service_info_json.contact_url = contact_url.clone(); + if let Some(contact_url) = config.contact_url() { + service_info_json.contact_url = contact_url.to_string(); } - if let Some(documentation_url) = &config.documentation_url { - service_info_json.documentation_url = documentation_url.clone(); + if let Some(documentation_url) = config.documentation_url() { + service_info_json.documentation_url = documentation_url.to_string(); } - if let Some(created_at) = &config.created_at { - service_info_json.created_at = created_at.clone(); + if let Some(created_at) = config.created_at() { + service_info_json.created_at = created_at.to_string(); } - if let Some(updated_at) = &config.updated_at { - service_info_json.updated_at = updated_at.clone(); + if let Some(updated_at) = config.updated_at() { + service_info_json.updated_at = updated_at.to_string(); } - if let Some(environment) = &config.environment { - service_info_json.environment = environment.clone(); + if let Some(environment) = config.environment() { + service_info_json.environment = environment.to_string(); } service_info_json diff --git a/htsget-http-lambda/src/lib.rs b/htsget-http-lambda/src/lib.rs index 7a6c84367..a617f1d8b 100644 --- a/htsget-http-lambda/src/lib.rs +++ b/htsget-http-lambda/src/lib.rs @@ -4,19 +4,20 @@ use std::collections::HashMap; use std::sync::Arc; +use htsget_config::Class; use lambda_http::ext::RequestExt; use lambda_http::http::{Method, StatusCode, Uri}; -use lambda_http::tower::ServiceBuilder; -use lambda_http::{http, service_fn, Body, Request, Response}; +use lambda_http::tower::{ServiceBuilder, ServiceExt}; +use lambda_http::{http, service_fn, Body, Request, Response, Service}; use lambda_runtime::Error; use tracing::instrument; use tracing::{debug, info}; +use htsget_config::config::cors::CorsConfig; +pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig}; #[cfg(feature = "s3-storage")] -pub use htsget_config::config::aws::AwsS3DataServer; -pub use htsget_config::config::{ - Config, LocalDataServer, ServiceInfo, StorageType, TicketServerConfig, -}; +pub use htsget_config::regex_resolver::aws::S3Resolver; +pub use htsget_config::regex_resolver::StorageType; use htsget_http_core::{Endpoint, PostRequest}; use htsget_search::htsget::HtsGet; use htsget_search::storage::configure_cors; @@ -176,15 +177,11 @@ impl<'a, H: HtsGet + Send + Sync + 'static> Router<'a, H> { } } -pub async fn handle_request( - cors_allow_credentials: bool, - cors_allow_origin: String, - router: &Router<'_, H>, -) -> Result<(), Error> +pub async fn handle_request(cors: CorsConfig, router: &Router<'_, H>) -> Result<(), Error> where H: HtsGet + Send + Sync + 'static, { - let cors_layer = configure_cors(cors_allow_credentials, cors_allow_origin)?; + let cors_layer = configure_cors(cors)?; let handler = ServiceBuilder::new() @@ -201,12 +198,14 @@ where #[cfg(test)] mod tests { + use super::*; use std::future::Future; use std::path::Path; use std::str::FromStr; use std::sync::Arc; use async_trait::async_trait; + use htsget_config::regex_resolver::{RegexResolver, StorageType, UrlResolver}; use htsget_config::Class; use lambda_http::http::header::HeaderName; use lambda_http::http::Uri; @@ -223,16 +222,13 @@ mod tests { use htsget_search::storage::data_server::HttpTicketFormatter; use htsget_search::storage::local::LocalStorage; use htsget_test_utils::http_tests::{config_with_tls, default_test_config, get_test_file}; - use htsget_test_utils::http_tests::{Header, Response, TestRequest, TestServer}; + use htsget_test_utils::http_tests::{Header, Response as TestResponse, TestRequest, TestServer}; use htsget_test_utils::server_tests::{ expected_url_path, formatter_and_expected_path, formatter_from_config, test_response, test_response_service_info, }; use htsget_test_utils::{cors_tests, server_tests}; - use crate::Config; - use crate::{service_fn, HtsgetMethod, Method, Route, RouteType, Router, ServiceBuilder}; - struct LambdaTestServer { config: Config, } @@ -300,15 +296,12 @@ mod tests { LambdaTestRequest(Request::default()) } - async fn test_server(&self, request: LambdaTestRequest) -> Response { + async fn test_server(&self, request: LambdaTestRequest) -> TestResponse { let (expected_path, formatter) = formatter_and_expected_path(self.get_config()).await; let router = Router::new( - Arc::new( - HtsGetFromStorage::local_from(&self.config.resolvers.first().unwrap().server, self.config.resolver.clone(), formatter) - .unwrap(), - ), - &self.config.ticket_server_config.service_info, + Arc::new(self.config.clone().owned_resolvers()), + self.config.ticket_server().service_info(), ); route_request_to_response(request.0, router, expected_path, &self.config).await @@ -656,14 +649,12 @@ mod tests { async fn with_router<'a, F, Fut>(test: F, config: &'a Config, formatter: HttpTicketFormatter) where - F: FnOnce(Router<'a, HtsGetFromStorage>>) -> Fut, + F: FnOnce(Router<'a, Vec>) -> Fut, Fut: Future, { let router = Router::new( - Arc::new( - HtsGetFromStorage::local_from(&config.path, config.resolver.clone(), formatter).unwrap(), - ), - &config.ticket_server_config.service_info, + Arc::new(config.clone().owned_resolvers()), + config.ticket_server().service_info(), ); test(router).await; } @@ -717,18 +708,9 @@ mod tests { router: Router<'_, T>, expected_path: String, config: &Config, - ) -> Response { + ) -> TestResponse { let response = ServiceBuilder::new() - .layer( - configure_cors( - config.data_server_config.data_server_cors_allow_credentials, - config - .data_server_config - .data_server_cors_allow_origin - .clone(), - ) - .unwrap(), - ) + .layer(configure_cors(config.ticket_server().cors().clone()).unwrap()) .service(service_fn(|event: Request| async { router.route_request(event).await })) @@ -742,6 +724,6 @@ mod tests { let status: u16 = response.status().into(); let body = response.body().to_vec(); - Response::new(status, response.headers().clone(), body, expected_path) + TestResponse::new(status, response.headers().clone(), body, expected_path) } } diff --git a/htsget-http-lambda/src/main.rs b/htsget-http-lambda/src/main.rs index 894904f6e..2df2d3e08 100644 --- a/htsget-http-lambda/src/main.rs +++ b/htsget-http-lambda/src/main.rs @@ -1,10 +1,9 @@ use std::sync::Arc; +use htsget_config::config::{DataServerConfig, TicketServerConfig}; +use htsget_config::regex_resolver::RegexResolver; use lambda_http::Error; use tracing::instrument; -use htsget_config::config::{LocalDataServer, TicketServerConfig}; -use htsget_config::config::aws::AwsS3DataServer; -use htsget_config::regex_resolver::RegexResolver; use htsget_http_lambda::{handle_request, Router}; use htsget_http_lambda::{Config, StorageType}; @@ -17,41 +16,9 @@ async fn main() -> Result<(), Error> { Config::setup_tracing()?; let config = Config::from_env(Config::parse_args())?; - let resolver = config.resolvers.first().unwrap(); - match resolver.server.clone() { - StorageType::LocalStorage(server_config) => local_storage_server(&server_config, resolver, config.ticket_server_config).await, - #[cfg(feature = "s3-storage")] - StorageType::AwsS3Storage(server_config) => s3_storage_server(&server_config, resolver, config.ticket_server_config).await, - _ => Err("unsupported storage type".into()), - } -} - -#[instrument(skip_all)] -async fn local_storage_server(config: &LocalDataServer, resolver: &RegexResolver, ticket_config: TicketServerConfig) -> Result<(), Error> { - let formatter = HttpTicketFormatter::try_from(config.clone())?; - let searcher: Arc>> = Arc::new( - HtsGetFromStorage::local_from(config.path.clone(), resolver.clone(), formatter)?, - ); - let router = &Router::new(searcher, &ticket_config.service_info); - - handle_request( - config.cors_allow_credentials, - config.cors_allow_origin.clone(), - router, - ) - .await -} - -#[cfg(feature = "s3-storage")] -#[instrument(skip_all)] -async fn s3_storage_server(config: &AwsS3DataServer, resolver: &RegexResolver, ticket_config: TicketServerConfig) -> Result<(), Error> { - let searcher = Arc::new(HtsGetFromStorage::s3_from(config.bucket.clone(), resolver.clone()).await); - let router = &Router::new(searcher, &ticket_config.service_info); + let service_info = config.ticket_server().service_info().clone(); + let cors = config.ticket_server().cors().clone(); + let router = &Router::new(Arc::new(config.owned_resolvers()), &service_info); - handle_request( - config.cors_allow_credentials, - config.cors_allow_origin.clone(), - router, - ) - .await + handle_request(cors, router).await } diff --git a/htsget-search/benches/search_benchmarks.rs b/htsget-search/benches/search_benchmarks.rs index 91d195433..7b831934b 100644 --- a/htsget-search/benches/search_benchmarks.rs +++ b/htsget-search/benches/search_benchmarks.rs @@ -4,8 +4,8 @@ use criterion::measurement::WallTime; use criterion::{criterion_group, criterion_main, BenchmarkGroup, Criterion}; use tokio::runtime::Runtime; -use htsget_config::config::StorageTypeServer; -use htsget_config::regex_resolver::MatchOnQuery; +use htsget_config::config::cors::CorsConfig; +use htsget_config::regex_resolver::StorageType; use htsget_config::Class::Header; use htsget_config::Format::{Bam, Bcf, Cram, Vcf}; use htsget_config::Query; @@ -21,17 +21,10 @@ const NUMBER_OF_SAMPLES: usize = 150; async fn perform_query(query: Query) -> Result<(), HtsGetError> { let htsget = HtsGetFromStorage::local_from( "../data", - RegexResolver::new( - ".*", - "$0", - StorageTypeServer::default(), - MatchOnQuery::default(), - ) - .unwrap(), + RegexResolver::new(StorageType::default(), ".*", "$0", Default::default()).unwrap(), HttpTicketFormatter::new( "127.0.0.1:8081".parse().expect("expected valid address"), - "".to_string(), - false, + CorsConfig::default(), ), )?; diff --git a/htsget-search/src/htsget/cram_search.rs b/htsget-search/src/htsget/cram_search.rs index 767a0dc52..470f56a2d 100644 --- a/htsget-search/src/htsget/cram_search.rs +++ b/htsget-search/src/htsget/cram_search.rs @@ -194,7 +194,7 @@ where let owned_record = record.clone(); let owned_next = next.clone(); let owned_predicate = predicate.clone(); - let range = query.interval.clone(); + let range = query.interval().clone(); futures.push_back(tokio::spawn(async move { if owned_predicate(&owned_record) { Self::bytes_ranges_for_record(range, &owned_record, owned_next.offset()) @@ -224,7 +224,7 @@ where } Some(last) if predicate(last) => { if let Some(range) = Self::bytes_ranges_for_record( - query.interval.clone(), + query.interval().clone(), last, self.position_at_eof(query).await?, )? { diff --git a/htsget-search/src/htsget/from_storage.rs b/htsget-search/src/htsget/from_storage.rs index 62549293a..eb587b14b 100644 --- a/htsget-search/src/htsget/from_storage.rs +++ b/htsget-search/src/htsget/from_storage.rs @@ -5,16 +5,18 @@ use std::path::Path; use std::sync::Arc; use async_trait::async_trait; +use htsget_config::regex_resolver::{Resolver, StorageType}; use tokio::io::AsyncRead; use tracing::debug; use tracing::instrument; use crate::htsget::search::Search; -use crate::htsget::Format; +use crate::htsget::{Format, HtsGetError}; #[cfg(feature = "s3-storage")] use crate::storage::aws::AwsS3Storage; +use crate::storage::data_server::HttpTicketFormatter; use crate::storage::local::LocalStorage; -use crate::storage::UrlFormatter; +use crate::storage::{StorageError, UrlFormatter}; use crate::RegexResolver; use crate::{ htsget::bam_search::BamSearch, @@ -31,6 +33,41 @@ pub struct HtsGetFromStorage { storage_ref: Arc, } +#[async_trait] +impl HtsGet for Vec { + async fn search(&self, query: Query) -> Result { + self.as_slice().search(query).await + } +} + +#[async_trait] +impl HtsGet for &[RegexResolver] { + async fn search(&self, query: Query) -> Result { + for resolver in self.iter() { + if let Some(id) = resolver.resolve_id(&query) { + match resolver.storage_type() { + StorageType::Url(url) => { + let searcher = + HtsGetFromStorage::local_from(url.path(), resolver.clone(), url.clone())?; + return searcher.search(query).await; + } + #[cfg(feature = "s3-storage")] + StorageType::S3(s3) => { + let searcher = + HtsGetFromStorage::s3_from(s3.bucket().to_string(), resolver.clone()).await; + return searcher.search(query).await; + } + _ => {} + } + } + } + + Err(HtsGetError::not_found( + "failed to match query with resolver", + )) + } +} + #[async_trait] impl HtsGet for HtsGetFromStorage where @@ -39,26 +76,14 @@ where { #[instrument(level = "debug", skip(self))] async fn search(&self, query: Query) -> Result { - debug!(?query.format, ?query, "searching {:?}, with query {:?}", query.format, query); - match query.format { + debug!(format = ?query.format(), ?query, "searching {:?}, with query {:?}", query.format(), query); + match query.format() { Format::Bam => BamSearch::new(self.storage()).search(query).await, Format::Cram => CramSearch::new(self.storage()).search(query).await, Format::Vcf => VcfSearch::new(self.storage()).search(query).await, Format::Bcf => BcfSearch::new(self.storage()).search(query).await, } } - - fn get_supported_formats(&self) -> Vec { - vec![Format::Bam, Format::Cram, Format::Vcf, Format::Bcf] - } - - fn are_field_parameters_effective(&self) -> bool { - false - } - - fn are_tag_parameters_effective(&self) -> bool { - false - } } impl HtsGetFromStorage { @@ -98,10 +123,10 @@ pub(crate) mod tests { use std::future::Future; use std::path::PathBuf; + use htsget_config::config::cors::CorsConfig; use tempfile::TempDir; - use htsget_config::config::StorageType; - use htsget_config::regex_resolver::MatchOnQuery; + use htsget_config::regex_resolver::StorageType; use htsget_test_utils::util::expected_bgzf_eof_data_url; use crate::htsget::bam_search::tests::{ @@ -180,14 +205,8 @@ pub(crate) mod tests { test(Arc::new( LocalStorage::new( base_path, - RegexResolver::new( - ".*", - "$0", - StorageType::default(), - MatchOnQuery::default(), - ) - .unwrap(), - HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), "".to_string(), false), + RegexResolver::new(Default::default(), ".*", "$0", Default::default()).unwrap(), + HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), CorsConfig::default()), ) .unwrap(), )) diff --git a/htsget-search/src/htsget/mod.rs b/htsget-search/src/htsget/mod.rs index 66bb06b9e..2c8058ef3 100644 --- a/htsget-search/src/htsget/mod.rs +++ b/htsget-search/src/htsget/mod.rs @@ -28,9 +28,18 @@ type Result = core::result::Result; #[async_trait] pub trait HtsGet { async fn search(&self, query: Query) -> Result; - fn get_supported_formats(&self) -> Vec; - fn are_field_parameters_effective(&self) -> bool; - fn are_tag_parameters_effective(&self) -> bool; + + fn get_supported_formats(&self) -> Vec { + vec![Format::Bam, Format::Cram, Format::Vcf, Format::Bcf] + } + + fn are_field_parameters_effective(&self) -> bool { + false + } + + fn are_tag_parameters_effective(&self) -> bool { + false + } } #[derive(Error, Debug, PartialEq, Eq)] @@ -281,37 +290,37 @@ mod tests { #[test] fn query_new() { let result = Query::new("NA12878", Format::Bam); - assert_eq!(result.id, "NA12878"); + assert_eq!(result.id(), "NA12878"); } #[test] fn query_with_format() { let result = Query::new("NA12878", Format::Bam); - assert_eq!(result.format, Format::Bam); + assert_eq!(result.format(), Format::Bam); } #[test] fn query_with_class() { let result = Query::new("NA12878", Format::Bam).with_class(Class::Header); - assert_eq!(result.class, Class::Header); + assert_eq!(result.class(), Class::Header); } #[test] fn query_with_reference_name() { let result = Query::new("NA12878", Format::Bam).with_reference_name("chr1"); - assert_eq!(result.reference_name, Some("chr1".to_string())); + assert_eq!(result.reference_name(), Some("chr1")); } #[test] fn query_with_start() { let result = Query::new("NA12878", Format::Bam).with_start(0); - assert_eq!(result.interval.start, Some(0)); + assert_eq!(result.interval().start(), Some(0)); } #[test] fn query_with_end() { let result = Query::new("NA12878", Format::Bam).with_end(0); - assert_eq!(result.interval.end, Some(0)); + assert_eq!(result.interval().end(), Some(0)); } #[test] @@ -319,23 +328,23 @@ mod tests { let result = Query::new("NA12878", Format::Bam) .with_fields(Fields::List(vec!["QNAME".to_string(), "FLAG".to_string()])); assert_eq!( - result.fields, - Fields::List(vec!["QNAME".to_string(), "FLAG".to_string()]) + result.fields(), + &Fields::List(vec!["QNAME".to_string(), "FLAG".to_string()]) ); } #[test] fn query_with_tags() { let result = Query::new("NA12878", Format::Bam).with_tags(Tags::All); - assert_eq!(result.tags, Tags::All); + assert_eq!(result.tags(), &Tags::All); } #[test] fn query_with_no_tags() { let result = Query::new("NA12878", Format::Bam).with_no_tags(vec!["RG", "OQ"]); assert_eq!( - result.no_tags, - NoTags(Some(vec!["RG".to_string(), "OQ".to_string()])) + result.no_tags(), + &NoTags(Some(vec!["RG".to_string(), "OQ".to_string()])) ); } diff --git a/htsget-search/src/htsget/search.rs b/htsget-search/src/htsget/search.rs index 80df3e653..b9d19e944 100644 --- a/htsget-search/src/htsget/search.rs +++ b/htsget-search/src/htsget/search.rs @@ -225,24 +225,30 @@ where /// Search based on the query. async fn search(&self, query: Query) -> Result { - match query.class { + match query.class() { Body => { let format = self.get_format(); - if format != query.format { + if format != query.format() { return Err(HtsGetError::unsupported_format(format!( "using `{}` search, but query contains `{}` format", - format, query.format + format, + query.format() ))); } - let byte_ranges = match query.reference_name.as_ref() { + let byte_ranges = match query.reference_name().as_ref() { None => self.get_byte_ranges_for_all(&query).await?, Some(reference_name) => { let index = self.read_index(&query).await?; let header = self.get_header(&query, &index).await?; let mut byte_ranges = self - .get_byte_ranges_for_reference_name(reference_name.clone(), &index, &header, &query) + .get_byte_ranges_for_reference_name( + reference_name.to_string(), + &index, + &header, + &query, + ) .await?; byte_ranges.push(self.get_byte_ranges_for_header(&index).await?); @@ -299,7 +305,7 @@ where else => break } } - return Ok(Response::new(query.format, urls)); + return Ok(Response::new(query.format(), urls)); } /// Get the header from the file specified by the id and format. @@ -385,9 +391,9 @@ where index: &Index, ) -> Result> { let chunks: Result> = trace_span!("querying chunks").in_scope(|| { - trace!(id = ?query.id.as_str(), ref_seq_id = ?ref_seq_id, "querying chunks"); + trace!(id = ?query.id(), ref_seq_id = ?ref_seq_id, "querying chunks"); let mut chunks = index - .query(ref_seq_id, query.interval.clone().into_one_based()?) + .query(ref_seq_id, query.interval().clone().into_one_based()?) .map_err(|err| HtsGetError::InvalidRange(format!("querying range: {}", err)))?; if chunks.is_empty() { @@ -396,7 +402,7 @@ where )); } - trace!(id = ?query.id.as_str(), ref_seq_id = ?ref_seq_id, "sorting chunks"); + trace!(id = ?query.id(), ref_seq_id = ?ref_seq_id, "sorting chunks"); chunks.sort_unstable_by_key(|a| a.end().compressed()); Ok(chunks) @@ -407,7 +413,7 @@ where Ok(gzi_data) => { let span = trace_span!("reading gzi"); let gzi: Result> = async { - trace!(id = ?query.id.as_str(), "reading gzi"); + trace!(id = ?query.id(), "reading gzi"); let mut gzi: Vec = gzi::AsyncReader::new(BufReader::new(gzi_data)) .read_index() .await? @@ -415,7 +421,7 @@ where .map(|(compressed, _)| compressed) .collect(); - trace!(id = ?query.id.as_str(), "sorting gzi"); + trace!(id = ?query.id(), "sorting gzi"); gzi.sort_unstable(); Ok(gzi) } diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index befb0087b..afc044e6d 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -1,9 +1,9 @@ +pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig}; #[cfg(feature = "s3-storage")] -pub use htsget_config::config::aws::AwsS3DataServer; -pub use htsget_config::config::{ - Config, LocalDataServer, ServiceInfo, StorageType, TicketServerConfig, +pub use htsget_config::regex_resolver::aws::S3Resolver; +pub use htsget_config::regex_resolver::{ + QueryMatcher, RegexResolver, Resolver, StorageType, UrlResolver, }; -pub use htsget_config::regex_resolver::{Resolver, RegexResolver}; pub mod htsget; pub mod storage; diff --git a/htsget-search/src/storage/aws.rs b/htsget-search/src/storage/aws.rs index 6f0e527d6..c148b4b38 100644 --- a/htsget-search/src/storage/aws.rs +++ b/htsget-search/src/storage/aws.rs @@ -76,10 +76,10 @@ impl AwsS3Storage { response .presigned( PresigningConfig::expires_in(Duration::from_secs(Self::PRESIGNED_REQUEST_EXPIRY)) - .map_err(|err| AwsS3Error(err.to_string(), query.id.to_string()))?, + .map_err(|err| AwsS3Error(err.to_string(), query.id().to_string()))?, ) .await - .map_err(|err| AwsS3Error(err.to_string(), query.id.to_string()))? + .map_err(|err| AwsS3Error(err.to_string(), query.id().to_string()))? .uri() .to_string(), ) @@ -93,7 +93,7 @@ impl AwsS3Storage { .key(resolve_id(&self.id_resolver, query)?) .send() .await - .map_err(|err| AwsS3Error(err.to_string(), query.id.to_string())) + .map_err(|err| AwsS3Error(err.to_string(), query.id().to_string())) } /// Returns the retrieval type of the object stored with the key. @@ -142,7 +142,7 @@ impl AwsS3Storage { if let Delayed(class) = self.get_retrieval_type(query).await? { return Err(AwsS3Error( format!("cannot retrieve object immediately, class is `{:?}`", class), - query.id.to_string(), + query.id().to_string(), )); } @@ -156,7 +156,7 @@ impl AwsS3Storage { response .send() .await - .map_err(|err| AwsS3Error(err.to_string(), query.id.to_string()))? + .map_err(|err| AwsS3Error(err.to_string(), query.id().to_string()))? .body, ) } @@ -178,7 +178,7 @@ impl Storage for AwsS3Storage { /// Gets the actual s3 object as a buffered reader. #[instrument(level = "trace", skip(self))] async fn get(&self, query: &Query, options: GetOptions) -> Result { - debug!(calling_from = ?self, query.id, "getting file with key {:?}", query.id); + debug!(calling_from = ?self, id = query.id(), "getting file with key {:?}", query.id()); self.create_stream_reader(query, options).await } @@ -189,7 +189,7 @@ impl Storage for AwsS3Storage { let presigned_url = self.s3_presign_url(query, options.range.clone()).await?; let url = options.apply(Url::new(presigned_url)); - debug!(calling_from = ?self, query.id, ?url, "getting url with key {:?}", query.id); + debug!(calling_from = ?self, id = query.id(), ?url, "getting url with key {:?}", query.id()); Ok(url) } @@ -204,7 +204,7 @@ impl Storage for AwsS3Storage { ) })?; - debug!(calling_from = ?self, query.id, len, "size of key {:?} is {}", query.id, len); + debug!(calling_from = ?self, id = query.id(), len, "size of key {:?} is {}", query.id(), len); Ok(len) } } @@ -225,8 +225,7 @@ mod tests { use s3_server::storages::fs::FileSystem; use s3_server::{S3Service, SimpleAuth}; - use htsget_config::config::StorageType; - use htsget_config::regex_resolver::MatchOnQuery; + use htsget_config::regex_resolver::UrlResolver; use htsget_config::Format::Bam; use htsget_config::Query; @@ -284,13 +283,7 @@ mod tests { test(AwsS3Storage::new( client, folder_name, - RegexResolver::new( - ".*", - "$0", - StorageType::default(), - MatchOnQuery::default(), - ) - .unwrap(), + RegexResolver::new(Default::default(), ".*", "$0", Default::default()).unwrap(), )); }) .await; diff --git a/htsget-search/src/storage/data_server.rs b/htsget-search/src/storage/data_server.rs index 05f632fd3..7ee5c9678 100644 --- a/htsget-search/src/storage/data_server.rs +++ b/htsget-search/src/storage/data_server.rs @@ -15,6 +15,9 @@ use axum::http; use axum::Router; use axum_extra::routing::SpaRouter; use futures_util::future::poll_fn; +use htsget_config::config::cors::CorsConfig; +use htsget_config::config::DataServerConfig; +use htsget_config::regex_resolver::UrlResolver; use http::uri::Scheme; use hyper::server::accept::Accept; use hyper::server::conn::{AddrIncoming, Http}; @@ -26,7 +29,6 @@ use tower::MakeService; use tower_http::trace::TraceLayer; use tracing::instrument; use tracing::{info, trace}; -use htsget_config::config::LocalDataServer; use crate::storage::StorageError::{DataServerError, IoError}; use crate::storage::{configure_cors, UrlFormatter}; @@ -50,30 +52,22 @@ pub struct HttpTicketFormatter { addr: SocketAddr, cert_key_pair: Option, scheme: Scheme, - cors_allow_origin: String, - cors_allow_credentials: bool, + cors: CorsConfig, } impl HttpTicketFormatter { const SERVE_ASSETS_AT: &'static str = "/data"; - pub fn new(addr: SocketAddr, cors_allow_origin: String, cors_allow_credentials: bool) -> Self { + pub fn new(addr: SocketAddr, cors: CorsConfig) -> Self { Self { addr, cert_key_pair: None, scheme: Scheme::HTTP, - cors_allow_origin, - cors_allow_credentials, + cors, } } - pub fn new_with_tls>( - addr: SocketAddr, - cors_allow_origin: String, - cors_allow_credentials: bool, - cert: P, - key: P, - ) -> Self { + pub fn new_with_tls>(addr: SocketAddr, cors: CorsConfig, cert: P, key: P) -> Self { Self { addr, cert_key_pair: Some(CertificateKeyPair { @@ -81,8 +75,7 @@ impl HttpTicketFormatter { key: PathBuf::from(key.as_ref()), }), scheme: Scheme::HTTPS, - cors_allow_origin, - cors_allow_credentials, + cors, } } @@ -98,8 +91,7 @@ impl HttpTicketFormatter { self.addr, Self::SERVE_ASSETS_AT, self.cert_key_pair.take(), - self.cors_allow_origin.clone(), - self.cors_allow_credentials, + self.cors.clone(), ) .await?; self.addr = server.local_addr(); @@ -112,28 +104,23 @@ impl HttpTicketFormatter { } } -impl TryFrom for HttpTicketFormatter { +impl TryFrom for HttpTicketFormatter { type Error = StorageError; /// Returns a ticket server with tls if both cert and key are not None, without tls if cert and key /// are both None, and otherwise an error. - fn try_from(config: LocalDataServer) -> Result { - match (config.cert, config.key) { + fn try_from(config: DataServerConfig) -> Result { + match (config.cert(), config.key()) { (Some(cert), Some(key)) => Ok(Self::new_with_tls( - config.addr, - config.cors_allow_origin, - config.cors_allow_credentials, + config.addr(), + config.cors().clone(), cert, key, )), (Some(_), None) | (None, Some(_)) => Err(DataServerError( "both the cert and key must be provided for the ticket server".to_string(), )), - (None, None) => Ok(Self::new( - config.addr, - config.cors_allow_origin, - config.cors_allow_credentials, - )), + (None, None) => Ok(Self::new(config.addr(), config.cors().clone())), } } } @@ -150,8 +137,7 @@ pub struct DataServer { listener: AddrIncoming, serve_assets_at: String, cert_key_pair: Option, - cors_allow_origin: String, - cors_allow_credentials: bool, + cors: CorsConfig, } impl DataServer { @@ -161,8 +147,7 @@ impl DataServer { addr: SocketAddr, serve_assets_at: impl Into, cert_key_pair: Option, - cors_allow_origin: String, - cors_allow_credentials: bool, + cors: CorsConfig, ) -> Result { let listener = TcpListener::bind(addr) .await @@ -174,8 +159,7 @@ impl DataServer { listener, serve_assets_at: serve_assets_at.into(), cert_key_pair, - cors_allow_origin, - cors_allow_credentials, + cors, }) } @@ -184,10 +168,7 @@ impl DataServer { pub async fn serve>(mut self, path: P) -> Result<()> { let mut app = Router::new() .merge(SpaRouter::new(&self.serve_assets_at, path)) - .layer(configure_cors( - self.cors_allow_credentials, - self.cors_allow_origin, - )?) + .layer(configure_cors(self.cors)?) .layer(TraceLayer::new_for_http()) .into_make_service_with_connect_info::(); @@ -416,7 +397,7 @@ mod tests { #[test] fn http_formatter_authority() { let formatter = - HttpTicketFormatter::new("127.0.0.1:8080".parse().unwrap(), "".to_string(), false); + HttpTicketFormatter::new("127.0.0.1:8080".parse().unwrap(), CorsConfig::default()); test_formatter_authority(formatter, "http"); } @@ -424,8 +405,7 @@ mod tests { fn https_formatter_authority() { let formatter = HttpTicketFormatter::new_with_tls( "127.0.0.1:8080".parse().unwrap(), - "".to_string(), - false, + CorsConfig::default(), "", "", ); @@ -435,7 +415,7 @@ mod tests { #[test] fn http_scheme() { let formatter = - HttpTicketFormatter::new("127.0.0.1:8080".parse().unwrap(), "".to_string(), false); + HttpTicketFormatter::new("127.0.0.1:8080".parse().unwrap(), CorsConfig::default()); assert_eq!(formatter.get_scheme(), &Scheme::HTTP); } @@ -443,8 +423,7 @@ mod tests { fn https_scheme() { let formatter = HttpTicketFormatter::new_with_tls( "127.0.0.1:8080".parse().unwrap(), - "".to_string(), - false, + CorsConfig::default(), "", "", ); @@ -454,7 +433,7 @@ mod tests { #[tokio::test] async fn get_addr_local_addr() { let mut formatter = - HttpTicketFormatter::new("127.0.0.1:0".parse().unwrap(), "".to_string(), false); + HttpTicketFormatter::new("127.0.0.1:0".parse().unwrap(), CorsConfig::default()); let server = formatter.bind_data_server().await.unwrap(); assert_eq!(formatter.get_addr(), server.local_addr()); } @@ -490,15 +469,9 @@ mod tests { P: AsRef + Send + 'static, { let addr = SocketAddr::from_str(&format!("{}:{}", "127.0.0.1", "0")).unwrap(); - let server = DataServer::bind_addr( - addr, - "/data", - cert_key_pair, - "http://example.com".to_string(), - false, - ) - .await - .unwrap(); + let server = DataServer::bind_addr(addr, "/data", cert_key_pair, CorsConfig::default()) + .await + .unwrap(); let port = server.local_addr().port(); tokio::spawn(async move { server.serve(path).await.unwrap() }); diff --git a/htsget-search/src/storage/local.rs b/htsget-search/src/storage/local.rs index 577c1644a..6bab3a15c 100644 --- a/htsget-search/src/storage/local.rs +++ b/htsget-search/src/storage/local.rs @@ -53,18 +53,18 @@ impl LocalStorage { .base_path .join(resolve_id(&self.id_resolver, query)?) .canonicalize() - .map_err(|_| StorageError::InvalidKey(query.id.to_string())) + .map_err(|_| StorageError::InvalidKey(query.id().to_string())) .and_then(|path| { path .starts_with(&self.base_path) .then_some(path) - .ok_or_else(|| StorageError::InvalidKey(query.id.to_string())) + .ok_or_else(|| StorageError::InvalidKey(query.id().to_string())) }) .and_then(|path| { path .is_file() .then_some(path) - .ok_or_else(|| StorageError::KeyNotFound(query.id.to_string())) + .ok_or_else(|| StorageError::KeyNotFound(query.id().to_string())) }) } @@ -72,7 +72,7 @@ impl LocalStorage { let path = self.get_path_from_key(query)?; File::open(path) .await - .map_err(|_| StorageError::KeyNotFound(query.id.to_string())) + .map_err(|_| StorageError::KeyNotFound(query.id().to_string())) } } @@ -83,7 +83,7 @@ impl Storage for LocalStorage { /// Get the file at the location of the key. #[instrument(level = "debug", skip(self))] async fn get(&self, query: &Query, _options: GetOptions) -> Result { - debug!(calling_from = ?self, id = query.id, "getting file with key {:?}", query.id); + debug!(calling_from = ?self, id = query.id(), "getting file with key {:?}", query.id()); self.get(query).await } @@ -99,7 +99,7 @@ impl Storage for LocalStorage { let url = Url::new(self.url_formatter.format_url(&path)?); let url = options.apply(url); - debug!(calling_from = ?self, id = query.id, ?url, "getting url with key {:?}", query.id); + debug!(calling_from = ?self, id = query.id(), ?url, "getting url with key {:?}", query.id()); Ok(url) } @@ -112,7 +112,7 @@ impl Storage for LocalStorage { .map_err(|err| StorageError::KeyNotFound(err.to_string()))? .len(); - debug!(calling_from = ?self, id = query.id, len, "size of key {:?} is {}", query.id, len); + debug!(calling_from = ?self, id = query.id(), len, "size of key {:?} is {}", query.id(), len); Ok(len) } } @@ -122,12 +122,12 @@ pub(crate) mod tests { use std::future::Future; use std::matches; + use htsget_config::config::cors::CorsConfig; use tempfile::TempDir; use tokio::fs::{create_dir, File}; use tokio::io::AsyncWriteExt; - use htsget_config::config::StorageType; - use htsget_config::regex_resolver::MatchOnQuery; + use htsget_config::regex_resolver::StorageType; use htsget_config::Format::Bam; use crate::htsget::{Headers, Url}; @@ -321,14 +321,8 @@ pub(crate) mod tests { test( LocalStorage::new( base_path.path(), - RegexResolver::new( - ".*", - "$0", - StorageType::default(), - MatchOnQuery::default(), - ) - .unwrap(), - HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), "".to_string(), false), + RegexResolver::new(StorageType::default(), ".*", "$0", Default::default()).unwrap(), + HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), CorsConfig::default()), ) .unwrap(), ) diff --git a/htsget-search/src/storage/mod.rs b/htsget-search/src/storage/mod.rs index d7fe968c8..51f377d78 100644 --- a/htsget-search/src/storage/mod.rs +++ b/htsget-search/src/storage/mod.rs @@ -9,17 +9,19 @@ use std::time::Duration; use async_trait::async_trait; use base64::encode; +use htsget_config::config::cors::{AllowType, CorsConfig, TaggedAllowTypes, TaggedAnyAllowType}; +use htsget_config::regex_resolver::{Scheme, UrlResolver}; use htsget_config::{Class, Query}; -use http::{HeaderValue, Method}; +use http::{uri, HeaderValue, Method}; use thiserror::Error; use tokio::io::AsyncRead; -use tower_http::cors::{AllowHeaders, AllowMethods, CorsLayer}; +use tower_http::cors::{AllowHeaders, AllowMethods, AllowOrigin, CorsLayer, ExposeHeaders}; use tracing::instrument; use crate::htsget::{Headers, Url}; use crate::storage::data_server::CORS_MAX_AGE; use crate::storage::StorageError::DataServerError; -use crate::{Resolver, RegexResolver}; +use crate::{RegexResolver, Resolver}; #[cfg(feature = "s3-storage")] pub mod aws; @@ -45,7 +47,10 @@ pub trait Storage { /// Get the url of the object using an inline data uri. #[instrument(level = "trace", ret)] - fn data_url(data: Vec, class: Option) -> Url { + fn data_url(data: Vec, class: Option) -> Url + where + Self: Sized, + { Url::new(format!("data:;base64,{}", encode(data))).set_class(class) } } @@ -87,33 +92,63 @@ pub enum StorageError { AwsS3Error(String, String), } +impl UrlFormatter for UrlResolver { + fn format_url>(&self, key: K) -> Result { + uri::Builder::new() + .scheme(match self.scheme() { + Scheme::Http => uri::Scheme::HTTP, + Scheme::Https => uri::Scheme::HTTPS, + }) + .authority(self.authority().to_string()) + .path_and_query(format!("{}/{}", self.path(), key.as_ref())) + .build() + .map_err(|err| StorageError::InvalidUri(err.to_string())) + .map(|value| value.to_string()) + } +} + /// Configure cors, settings allowed methods, max age, allowed origins, and if credentials /// are supported. -pub fn configure_cors( - cors_allow_credentials: bool, - cors_allow_origin: String, -) -> Result { +pub fn configure_cors(cors: CorsConfig) -> Result { + let mut cors_layer = CorsLayer::new(); + cors_layer = match cors.allow_origins() { + AllowType::Tagged(tagged) => match tagged { + TaggedAllowTypes::Mirror => cors_layer.allow_origin(AllowOrigin::mirror_request()), + TaggedAllowTypes::Any => cors_layer.allow_origin(AllowOrigin::any()), + }, + AllowType::List(origins) => cors_layer.allow_origin( + origins + .iter() + .map(|header| header.clone().into_inner()) + .collect::>(), + ), + }; + + cors_layer = match cors.allow_headers() { + AllowType::Tagged(tagged) => match tagged { + TaggedAllowTypes::Mirror => cors_layer.allow_headers(AllowHeaders::mirror_request()), + TaggedAllowTypes::Any => cors_layer.allow_headers(AllowHeaders::any()), + }, + AllowType::List(headers) => cors_layer.allow_headers(headers.clone()), + }; + + cors_layer = match cors.allow_methods() { + AllowType::Tagged(tagged) => match tagged { + TaggedAllowTypes::Mirror => cors_layer.allow_methods(AllowMethods::mirror_request()), + TaggedAllowTypes::Any => cors_layer.allow_methods(AllowMethods::any()), + }, + AllowType::List(methods) => cors_layer.allow_methods(methods.clone()), + }; + + cors_layer = match cors.expose_headers() { + AllowType::Tagged(_) => cors_layer, + AllowType::List(headers) => cors_layer.expose_headers(headers.clone()), + }; + Ok( - CorsLayer::new() - .allow_origin( - cors_allow_origin - .parse::() - .map_err(|err| DataServerError(format!("failed parsing allowed origin: `{}`", err)))?, - ) - .allow_headers(AllowHeaders::mirror_request()) - .max_age(Duration::from_secs(CORS_MAX_AGE)) - .allow_credentials(cors_allow_credentials) - .allow_methods(AllowMethods::list(vec![ - Method::GET, - Method::POST, - Method::PUT, - Method::DELETE, - Method::HEAD, - Method::OPTIONS, - Method::CONNECT, - Method::PATCH, - Method::TRACE, - ])), + cors_layer + .allow_credentials(cors.allow_credentials()) + .max_age(Duration::from_secs(cors.max_age() as u64)), ) } @@ -371,7 +406,7 @@ impl RangeUrlOptions { fn resolve_id(resolver: &RegexResolver, query: &Query) -> Result { resolver .resolve_id(query) - .ok_or_else(|| StorageError::InvalidKey(query.id.to_string())) + .ok_or_else(|| StorageError::InvalidKey(query.id().to_string())) } #[cfg(test)] diff --git a/htsget-test-utils/src/http_tests.rs b/htsget-test-utils/src/http_tests.rs index 2e204f0f2..78e4e8f93 100644 --- a/htsget-test-utils/src/http_tests.rs +++ b/htsget-test-utils/src/http_tests.rs @@ -2,10 +2,11 @@ use std::fs; use std::path::{Path, PathBuf}; use async_trait::async_trait; +use htsget_config::config::cors::{AllowType, CorsConfig}; +use htsget_config::config::DataServerConfig; +use htsget_config::regex_resolver::RegexResolver; use http::HeaderMap; use serde::de; -use htsget_config::config::{LocalDataServer, StorageType}; -use htsget_config::regex_resolver::RegexResolver; use crate::util::generate_test_certificates; use crate::Config; @@ -85,46 +86,61 @@ pub fn default_dir_data() -> PathBuf { default_dir().join("data") } -fn set_path(config: &mut LocalDataServer) { - config.path = default_dir_data(); +fn set_path(config: &mut DataServerConfig) { + config.set_path(default_dir_data()); } -fn set_addr_and_path(config: &mut LocalDataServer) { +fn set_addr_and_path(config: &mut DataServerConfig) { set_path(config); - config.addr = "127.0.0.1:0".parse().unwrap(); + config.set_addr("127.0.0.1:0".parse().unwrap()); } /// Default config with fixed port. -pub fn default_config_fixed_port() -> LocalDataServer { - let mut config = LocalDataServer::default(); - set_path(&mut config); +pub fn default_config_fixed_port() -> Config { + let mut config = Config::default(); + + let mut data_server_config = DataServerConfig::default(); + set_path(&mut data_server_config); + + config.set_data_server(Some(data_server_config)); + config } /// Default config using the current cargo manifest directory, and dynamic port. pub fn default_test_config() -> Config { - let mut server_config = LocalDataServer::default(); + let mut server_config = DataServerConfig::default(); set_addr_and_path(&mut server_config); - let mut server_config = LocalDataServer::default(); - server_config.cors_allow_credentials = false; - server_config.cors_allow_origin = "http://example.com".to_string(); + let mut server_config = DataServerConfig::default(); + let mut cors = CorsConfig::default(); + + cors.set_allow_credentials(false); + cors.set_allow_origins(AllowType::List(vec!["http://example.com".parse().unwrap()])); - Config::from(server_config) + server_config.set_cors(cors); + + let mut config = Config::default(); + config.set_data_server(Some(server_config)); + + config } /// Config with tls ticket server, using the current cargo manifest directory. pub fn config_with_tls>(path: P) -> Config { - let mut server_config = LocalDataServer::default(); + let mut server_config = DataServerConfig::default(); set_addr_and_path(&mut server_config); let (key_path, cert_path) = generate_test_certificates(path, "key.pem", "cert.pem"); - let mut server_config = LocalDataServer::default(); - server_config.key = Some(key_path); - server_config.cert = Some(cert_path); + let mut server_config = DataServerConfig::default(); + server_config.set_key(Some(key_path)); + server_config.set_cert(Some(cert_path)); - Config::from(server_config) + let mut config = Config::default(); + config.set_data_server(Some(server_config)); + + config } /// Get the event associated with the file. diff --git a/htsget-test-utils/src/lib.rs b/htsget-test-utils/src/lib.rs index 2dcd93bd6..8d47b573a 100644 --- a/htsget-test-utils/src/lib.rs +++ b/htsget-test-utils/src/lib.rs @@ -1,8 +1,12 @@ -#[cfg(feature = "s3-storage")] -pub use htsget_config::config::aws::AwsS3DataServer; +#[cfg(all( + feature = "s3-storage", + any(feature = "cors-tests", feature = "server-tests") +))] +pub use htsget_config::regex_resolver::aws::S3Resolver; #[cfg(any(feature = "cors-tests", feature = "server-tests"))] -pub use htsget_config::config::{ - Config, LocalDataServer, ServiceInfo, StorageType, TicketServerConfig, +pub use htsget_config::{ + config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig}, + regex_resolver::StorageType, }; #[cfg(feature = "cors-tests")] diff --git a/htsget-test-utils/src/server_tests.rs b/htsget-test-utils/src/server_tests.rs index 008c7b9da..ed3770b77 100644 --- a/htsget-test-utils/src/server_tests.rs +++ b/htsget-test-utils/src/server_tests.rs @@ -3,12 +3,12 @@ use std::path::PathBuf; use futures::future::join_all; use futures::TryStreamExt; +use htsget_config::regex_resolver::UrlResolver; use htsget_config::{Class, Format}; use http::Method; use noodles_bgzf as bgzf; use noodles_vcf as vcf; use reqwest::ClientBuilder; -use htsget_config::config::StorageType::LocalStorage; use htsget_http_core::{get_service_info_with, Endpoint}; use htsget_search::htsget::Response as HtsgetResponse; @@ -74,13 +74,7 @@ pub async fn test_response(response: Response, class: Class) { /// Create the a [HttpTicketFormatter], spawn the ticket server, returning the expected path and the formatter. pub async fn formatter_and_expected_path(config: &Config) -> (String, HttpTicketFormatter) { - let mut formatter = formatter_from_config(config).unwrap(); - for resolver in config.resolvers.iter() { - if let LocalStorage(server) = &resolver.server { - spawn_ticket_server(server.path.clone(), &mut formatter).await; - } - } - + let formatter = formatter_from_config(config).unwrap(); (expected_url_path(&formatter), formatter) } @@ -168,11 +162,7 @@ pub async fn test_parameterized_post_class_header(tester: &impl /// Get the [HttpTicketFormatter] from the config. pub fn formatter_from_config(config: &Config) -> Option { - if let LocalStorage(server_config) = config.resolvers.first()?.server.clone() { - HttpTicketFormatter::try_from(server_config).ok() - } else { - None - } + HttpTicketFormatter::try_from(config.data_server().unwrap().clone()).ok() } /// A service info test. From 7a28706cf475411fe1c7cfefb3e606f0c0eb6dac Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 20 Dec 2022 08:38:31 +1100 Subject: [PATCH 26/45] config: fix logic involving allowed attributes --- htsget-config/src/config/mod.rs | 2 +- htsget-config/src/lib.rs | 7 +- htsget-config/src/regex_resolver/mod.rs | 148 ++++++++++-------------- 3 files changed, 69 insertions(+), 88 deletions(-) diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index f04c5aa73..36d39085d 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -618,7 +618,7 @@ mod tests { "#, |config| { assert_eq!( - config.resolvers().first().unwrap().match_formats(), + config.resolvers().first().unwrap().allowed_formats(), &vec![Bam] ); }, diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 69a61a45c..d0610e771 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use std::fmt::Formatter; use std::io::ErrorKind::Other; use std::{fmt, io}; +use std::collections::HashSet; use tracing::instrument; pub mod config; @@ -172,7 +173,7 @@ pub enum Fields { #[serde(alias = "all", alias = "ALL")] All, /// List of fields to include - List(Vec), + List(HashSet), } /// Possible values for the tags parameter. @@ -183,12 +184,12 @@ pub enum Tags { #[serde(alias = "all", alias = "ALL")] All, /// List of tags to include - List(Vec), + List(HashSet), } /// The no tags parameter. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -pub struct NoTags(pub Option>); +pub struct NoTags(pub Option>); /// A query contains all the parameters that can be used when requesting /// a search for either of `reads` or `variants`. diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 248abad9e..78b2ab8df 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -7,6 +7,7 @@ use crate::config::{default_localstorage_addr, default_serve_at}; use crate::regex_resolver::aws::S3Resolver; use crate::Format::{Bam, Bcf, Cram, Vcf}; use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; +use crate::regex_resolver::ReferenceNames::All; #[cfg(feature = "s3-storage")] pub mod aws; @@ -105,66 +106,68 @@ pub struct RegexResolver { #[derive(Serialize, Clone, Debug, Deserialize)] #[serde(default)] pub struct QueryGuard { - match_formats: Vec, - match_class: Vec, + allowed_formats: Vec, + allowed_classes: Vec, + allowed_reference_names: ReferenceNames, + allowed_interval: Interval, + allowed_fields: Fields, + allowed_tags: Tags, +} + +/// Referneces names that can be matched. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ReferenceNames { + All, #[serde(with = "serde_regex")] - match_reference_name: Regex, - /// The start and end positions are 0-based. [start, end) - start_interval: Interval, - end_interval: Interval, - match_fields: Fields, - match_tags: Tags, - match_no_tags: NoTags, + Some(Regex) } impl QueryGuard { - pub fn match_formats(&self) -> &[Format] { - &self.match_formats - } - - pub fn match_classes(&self) -> &[Class] { - &self.match_class - } - - pub fn match_reference_name(&self) -> &Regex { - &self.match_reference_name + pub fn allowed_formats(&self) -> &[Format] { + &self.allowed_formats } - pub fn start_interval(&self) -> Interval { - self.start_interval + pub fn allowed_classes(&self) -> &[Class] { + &self.allowed_classes } - pub fn end_interval(&self) -> Interval { - self.end_interval + pub fn allowed_reference_names(&self) -> &ReferenceNames { + &self.allowed_reference_names } - pub fn match_fields(&self) -> &Fields { - &self.match_fields + pub fn allowed_interval(&self) -> Interval { + self.allowed_interval } - pub fn match_tags(&self) -> &Tags { - &self.match_tags + pub fn allowed_fields(&self) -> &Fields { + &self.allowed_fields } - pub fn match_no_tags(&self) -> &NoTags { - &self.match_no_tags + pub fn allowed_tags(&self) -> &Tags { + &self.allowed_tags } } impl Default for QueryGuard { fn default() -> Self { Self { - match_formats: vec![Bam, Cram, Vcf, Bcf], - match_class: vec![Class::Body, Class::Header], - match_reference_name: Regex::new(".*").expect("Expected valid regex expression"), - start_interval: Interval { - start: Some(0), - end: Some(100), - }, - end_interval: Default::default(), - match_fields: Fields::All, - match_tags: Tags::All, - match_no_tags: NoTags(None), + allowed_formats: vec![Bam, Cram, Vcf, Bcf], + allowed_classes: vec![Class::Body, Class::Header], + allowed_reference_names: All, + allowed_interval: Default::default(), + allowed_fields: Fields::All, + allowed_tags: Tags::All, + } + } +} + +impl QueryMatcher for ReferenceNames { + fn query_matches(&self, query: &Query) -> bool { + match (self, &query.reference_name) { + (ReferenceNames::All, _) => true, + (ReferenceNames::Some(regex), Some(reference_name)) => regex.is_match(reference_name), + (ReferenceNames::Some(_), None) => false, } } } @@ -173,7 +176,7 @@ impl QueryMatcher for Fields { fn query_matches(&self, query: &Query) -> bool { match (self, &query.fields) { (Fields::All, _) => true, - (Fields::List(self_fields), Fields::List(query_fields)) => self_fields == query_fields, + (Fields::List(self_fields), Fields::List(query_fields)) => self_fields.is_subset(query_fields), (Fields::List(_), Fields::All) => false, } } @@ -183,40 +186,25 @@ impl QueryMatcher for Tags { fn query_matches(&self, query: &Query) -> bool { match (self, &query.tags) { (Tags::All, _) => true, - (Tags::List(self_tags), Tags::List(query_tags)) => self_tags == query_tags, + (Tags::List(self_tags), Tags::List(query_tags)) => self_tags.is_subset(query_tags), (Tags::List(_), Tags::All) => false, } } } -impl QueryMatcher for NoTags { - fn query_matches(&self, query: &Query) -> bool { - match (self, &query.no_tags) { - (NoTags(None), _) => true, - (NoTags(Some(self_no_tags)), NoTags(Some(query_no_tags))) => self_no_tags == query_no_tags, - (NoTags(Some(_)), NoTags(None)) => false, - } - } -} - impl QueryMatcher for QueryGuard { fn query_matches(&self, query: &Query) -> bool { - if let Some(reference_name) = &query.reference_name { - self.match_formats.contains(&query.format) - && self.match_class.contains(&query.class) - && self.match_reference_name.is_match(reference_name) + self.allowed_formats.contains(&query.format) + && self.allowed_classes.contains(&query.class) + && self.allowed_reference_names.query_matches(query) && self - .start_interval + .allowed_interval .contains(query.interval.start.unwrap_or(u32::MIN)) && self - .end_interval + .allowed_interval .contains(query.interval.end.unwrap_or(u32::MAX)) - && self.match_fields.query_matches(query) - && self.match_tags.query_matches(query) - && self.match_no_tags.query_matches(query) - } else { - false - } + && self.allowed_fields.query_matches(query) + && self.allowed_tags.query_matches(query) } } @@ -259,36 +247,28 @@ impl RegexResolver { &self.storage_type } - pub fn match_formats(&self) -> &[Format] { - self.guard.match_formats() - } - - pub fn match_classes(&self) -> &[Class] { - self.guard.match_classes() - } - - pub fn match_reference_name(&self) -> &Regex { - &self.guard.match_reference_name + pub fn allowed_formats(&self) -> &[Format] { + self.guard.allowed_formats() } - pub fn start_interval(&self) -> Interval { - self.guard.start_interval + pub fn allowed_classes(&self) -> &[Class] { + self.guard.allowed_classes() } - pub fn end_interval(&self) -> Interval { - self.guard.end_interval + pub fn allowed_reference_names(&self) -> &ReferenceNames { + &self.guard.allowed_reference_names } - pub fn match_fields(&self) -> &Fields { - &self.guard.match_fields + pub fn allowed_interval(&self) -> Interval { + self.guard.allowed_interval } - pub fn match_tags(&self) -> &Tags { - &self.guard.match_tags + pub fn allowed_fields(&self) -> &Fields { + &self.guard.allowed_fields } - pub fn match_no_tags(&self) -> &NoTags { - &self.guard.match_no_tags + pub fn allowed_tags(&self) -> &Tags { + &self.guard.allowed_tags } } From c0a6e0f5c18ae86d2dcb484e18b7a59592578466 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 20 Dec 2022 10:10:46 +1100 Subject: [PATCH 27/45] config: remove custom deserializer for None option and instead use custom enum --- htsget-config/config.toml | 1 - htsget-config/src/config/cors.rs | 10 +-- htsget-config/src/config/mod.rs | 88 +++++++++++++++++++++---- htsget-config/src/regex_resolver/mod.rs | 86 ++++++++++++------------ 4 files changed, 123 insertions(+), 62 deletions(-) diff --git a/htsget-config/config.toml b/htsget-config/config.toml index cdbbe6d49..e74a53fdf 100644 --- a/htsget-config/config.toml +++ b/htsget-config/config.toml @@ -1,5 +1,4 @@ ticket_server_addr = "127.0.0.1:8082" -data_server = "None" #ticket_server_cors_allow_credentials = false #ticket_server_cors_allow_origin = "http://localhost:8080" #start_data_server = true diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs index adfbca7aa..9325e5d06 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/cors.rs @@ -188,7 +188,7 @@ mod tests { #[test] fn unit_variant_any_allow_type() { test_cors_config( - "cors_allow_methods = \"Any\"", + "allow_methods = \"Any\"", &AllowType::Tagged(TaggedAllowTypes::Any), |config| config.allow_methods(), ); @@ -197,7 +197,7 @@ mod tests { #[test] fn unit_variant_mirror_allow_type() { test_cors_config( - "cors_allow_methods = \"Mirror\"", + "allow_methods = \"Mirror\"", &AllowType::Tagged(TaggedAllowTypes::Mirror), |config| config.allow_methods(), ); @@ -206,7 +206,7 @@ mod tests { #[test] fn list_allow_type() { test_cors_config( - "cors_allow_methods = [\"GET\"]", + "allow_methods = [\"GET\"]", &AllowType::List(vec![Method::GET]), |config| config.allow_methods(), ); @@ -215,7 +215,7 @@ mod tests { #[test] fn tagged_any_allow_type() { test_cors_config( - "cors_expose_headers = \"Any\"", + "expose_headers = \"Any\"", &AllowType::Tagged(TaggedAnyAllowType::Any), |config| config.expose_headers(), ); @@ -223,7 +223,7 @@ mod tests { #[test] fn tagged_any_allow_type_err_on_mirror() { - let allow_type_method = "cors_expose_headers = \"Mirror\""; + let allow_type_method = "expose_headers = \"Mirror\""; let config: Result = toml::from_str(allow_type_method); assert!(matches!(config, Err(_))); } diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 36d39085d..d6c153769 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -3,6 +3,7 @@ pub mod cors; use std::fmt::Debug; use std::io; use std::io::ErrorKind; +use std::iter::Map; use std::net::SocketAddr; use std::path::{Path, PathBuf}; @@ -10,10 +11,11 @@ use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAnyAllowType use clap::Parser; use figment::providers::{Env, Format, Serialized, Toml}; use figment::Figment; +use figment::value::Value::Dict; use http::header::HeaderName; use http::Method; use serde::de::IntoDeserializer; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_with::with_prefix; use tracing::info; use tracing::instrument; @@ -89,13 +91,12 @@ struct Args { config: PathBuf, } -fn empty_string_as_none<'de, D, T>(deserializer: D) -> Result, D::Error> +fn deserialize_empty_string_as_none<'de, D, T>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, T: Deserialize<'de>, { - let optional_string = Option::deserialize(deserializer)? - .filter(|s: &String| !s.is_empty() && s.to_lowercase() != "none"); + let optional_string = Option::deserialize(deserializer)?.filter(|s: &String| !s.is_empty() && s.to_lowercase() != "none"); if let Some(string) = optional_string { Ok(Some(T::deserialize(string.into_deserializer())?)) } else { @@ -103,17 +104,42 @@ where } } +fn serialize_empty_string_as_none(optional: &Option, serializer: S) -> Result + where + S: Serializer, + T: Serialize, +{ + match optional { + None => serializer.serialize_str("None"), + Some(value) => T::serialize(value, serializer) + } +} + /// Configuration for the server. Each field will be read from environment variables. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct Config { #[serde(flatten)] ticket_server: TicketServerConfig, - #[serde(deserialize_with = "empty_string_as_none")] - data_server: Option, + data_server: DataServerConfigOption, resolvers: Vec, } +/// None component of data server config. Allows deserializing no data server config as none. +#[derive(Serialize, Deserialize, Debug, Clone)] +enum DataServerConfigNone { + #[serde(alias = "none", alias = "NONE", alias = "")] + None +} + +/// Data server config enum options. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged)] +enum DataServerConfigOption { + None(DataServerConfigNone), + Some(DataServerConfig) +} + with_prefix!(ticket_server_prefix "ticket_server_"); /// Configuration for the htsget server. @@ -379,7 +405,7 @@ impl Default for Config { fn default() -> Self { Self { ticket_server: TicketServerConfig::default(), - data_server: Some(DataServerConfig::default()), + data_server: DataServerConfigOption::Some(DataServerConfig::default()), resolvers: vec![RegexResolver::default(), RegexResolver::default()], } } @@ -428,7 +454,10 @@ impl Config { } pub fn data_server(&self) -> Option<&DataServerConfig> { - self.data_server.as_ref() + match self.data_server { + DataServerConfigOption::None(_) => None, + DataServerConfigOption::Some(ref config) => Some(config) + } } pub fn resolvers(&self) -> &[RegexResolver] { @@ -444,7 +473,14 @@ impl Config { } pub fn set_data_server(&mut self, data_server: Option) { - self.data_server = data_server; + match data_server { + None => { + self.data_server = DataServerConfigOption::None(DataServerConfigNone::None); + }, + Some(value ) => { + self.data_server = DataServerConfigOption::Some(value); + } + } } pub fn set_resolvers(&mut self, resolvers: Vec) { @@ -530,7 +566,7 @@ mod tests { #[test] fn config_data_server_addr_env() { test_config_from_env( - vec![("HTSGET_DATA_SERVERS", "[{addr=127.0.0.1:8082}]")], + vec![("HTSGET_DATA_SERVER", "{addr=127.0.0.1:8082}")], |config| { assert_eq!( config.data_server().unwrap().addr(), @@ -540,6 +576,19 @@ mod tests { ); } + #[test] + fn config_no_data_server_env() { + test_config_from_env( + vec![("HTSGET_DATA_SERVER", "")], + |config| { + assert!(matches!( + config.data_server(), + None + )); + }, + ); + } + #[test] fn config_resolvers_env() { test_config_from_env(vec![("HTSGET_RESOLVERS", "[{regex=regex}]")], |config| { @@ -578,7 +627,7 @@ mod tests { fn config_data_server_addr_file() { test_config_from_file( r#" - [[data_servers]] + [data_server] addr = "127.0.0.1:8082" "#, |config| { @@ -590,6 +639,19 @@ mod tests { ); } + #[test] + fn config_no_data_server_file() { + test_config_from_file( + r#"data_server = """#, + |config| { + assert!(matches!( + config.data_server(), + None + )); + }, + ); + } + #[test] fn config_resolvers_file() { test_config_from_file( @@ -614,11 +676,11 @@ mod tests { regex = "regex" [resolvers.guard] - match_formats = ["BAM"] + allow_formats = ["BAM"] "#, |config| { assert_eq!( - config.resolvers().first().unwrap().allowed_formats(), + config.resolvers().first().unwrap().allow_formats(), &vec![Bam] ); }, diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 78b2ab8df..b1a6178ce 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -106,12 +106,12 @@ pub struct RegexResolver { #[derive(Serialize, Clone, Debug, Deserialize)] #[serde(default)] pub struct QueryGuard { - allowed_formats: Vec, - allowed_classes: Vec, - allowed_reference_names: ReferenceNames, - allowed_interval: Interval, - allowed_fields: Fields, - allowed_tags: Tags, + allow_formats: Vec, + allow_classes: Vec, + allow_reference_names: ReferenceNames, + allow_interval: Interval, + allow_fields: Fields, + allow_tags: Tags, } /// Referneces names that can be matched. @@ -124,40 +124,40 @@ pub enum ReferenceNames { } impl QueryGuard { - pub fn allowed_formats(&self) -> &[Format] { - &self.allowed_formats + pub fn allow_formats(&self) -> &[Format] { + &self.allow_formats } - pub fn allowed_classes(&self) -> &[Class] { - &self.allowed_classes + pub fn allow_classes(&self) -> &[Class] { + &self.allow_classes } - pub fn allowed_reference_names(&self) -> &ReferenceNames { - &self.allowed_reference_names + pub fn allow_reference_names(&self) -> &ReferenceNames { + &self.allow_reference_names } - pub fn allowed_interval(&self) -> Interval { - self.allowed_interval + pub fn allow_interval(&self) -> Interval { + self.allow_interval } - pub fn allowed_fields(&self) -> &Fields { - &self.allowed_fields + pub fn allow_fields(&self) -> &Fields { + &self.allow_fields } - pub fn allowed_tags(&self) -> &Tags { - &self.allowed_tags + pub fn allow_tags(&self) -> &Tags { + &self.allow_tags } } impl Default for QueryGuard { fn default() -> Self { Self { - allowed_formats: vec![Bam, Cram, Vcf, Bcf], - allowed_classes: vec![Class::Body, Class::Header], - allowed_reference_names: All, - allowed_interval: Default::default(), - allowed_fields: Fields::All, - allowed_tags: Tags::All, + allow_formats: vec![Bam, Cram, Vcf, Bcf], + allow_classes: vec![Class::Body, Class::Header], + allow_reference_names: All, + allow_interval: Default::default(), + allow_fields: Fields::All, + allow_tags: Tags::All, } } } @@ -194,17 +194,17 @@ impl QueryMatcher for Tags { impl QueryMatcher for QueryGuard { fn query_matches(&self, query: &Query) -> bool { - self.allowed_formats.contains(&query.format) - && self.allowed_classes.contains(&query.class) - && self.allowed_reference_names.query_matches(query) + self.allow_formats.contains(&query.format) + && self.allow_classes.contains(&query.class) + && self.allow_reference_names.query_matches(query) && self - .allowed_interval + .allow_interval .contains(query.interval.start.unwrap_or(u32::MIN)) && self - .allowed_interval + .allow_interval .contains(query.interval.end.unwrap_or(u32::MAX)) - && self.allowed_fields.query_matches(query) - && self.allowed_tags.query_matches(query) + && self.allow_fields.query_matches(query) + && self.allow_tags.query_matches(query) } } @@ -247,28 +247,28 @@ impl RegexResolver { &self.storage_type } - pub fn allowed_formats(&self) -> &[Format] { - self.guard.allowed_formats() + pub fn allow_formats(&self) -> &[Format] { + self.guard.allow_formats() } - pub fn allowed_classes(&self) -> &[Class] { - self.guard.allowed_classes() + pub fn allow_classes(&self) -> &[Class] { + self.guard.allow_classes() } - pub fn allowed_reference_names(&self) -> &ReferenceNames { - &self.guard.allowed_reference_names + pub fn allow_reference_names(&self) -> &ReferenceNames { + &self.guard.allow_reference_names } - pub fn allowed_interval(&self) -> Interval { - self.guard.allowed_interval + pub fn allow_interval(&self) -> Interval { + self.guard.allow_interval } - pub fn allowed_fields(&self) -> &Fields { - &self.guard.allowed_fields + pub fn allow_fields(&self) -> &Fields { + &self.guard.allow_fields } - pub fn allowed_tags(&self) -> &Tags { - &self.guard.allowed_tags + pub fn allow_tags(&self) -> &Tags { + &self.guard.allow_tags } } From fe3283b38f53be48472aecff76df53edda687065 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 20 Dec 2022 17:08:49 +1100 Subject: [PATCH 28/45] test: fix tests affected by config, change some default values and move around config options --- htsget-config/src/config/mod.rs | 65 +++++++---------- htsget-config/src/lib.rs | 5 ++ htsget-config/src/regex_resolver/mod.rs | 51 ++++++++----- htsget-http-actix/src/main.rs | 2 +- htsget-http-core/src/query_builder.rs | 19 ++--- htsget-http-lambda/src/lib.rs | 2 +- htsget-search/src/htsget/from_storage.rs | 8 +-- htsget-search/src/htsget/mod.rs | 7 +- htsget-search/src/htsget/search.rs | 10 +-- htsget-search/src/lib.rs | 2 +- htsget-search/src/storage/aws.rs | 92 ++++++++++++++---------- htsget-search/src/storage/data_server.rs | 6 +- htsget-search/src/storage/local.rs | 63 ++++++++-------- htsget-search/src/storage/mod.rs | 20 ++++-- htsget-test-utils/src/cors_tests.rs | 26 ++++--- htsget-test-utils/src/http_tests.rs | 68 ++++++++++++++---- htsget-test-utils/src/server_tests.rs | 14 ++-- 17 files changed, 274 insertions(+), 186 deletions(-) diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index d6c153769..d7e97eb63 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -75,7 +75,7 @@ fn default_server_origin() -> &'static str { "http://localhost:8080" } -fn default_path() -> &'static str { +pub(crate) fn default_path() -> &'static str { "data" } @@ -88,31 +88,7 @@ pub(crate) fn default_serve_at() -> &'static str { #[command(author, version, about, long_about = USAGE)] struct Args { #[arg(short, long, env = "HTSGET_CONFIG")] - config: PathBuf, -} - -fn deserialize_empty_string_as_none<'de, D, T>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, - T: Deserialize<'de>, -{ - let optional_string = Option::deserialize(deserializer)?.filter(|s: &String| !s.is_empty() && s.to_lowercase() != "none"); - if let Some(string) = optional_string { - Ok(Some(T::deserialize(string.into_deserializer())?)) - } else { - Ok(None) - } -} - -fn serialize_empty_string_as_none(optional: &Option, serializer: S) -> Result - where - S: Serializer, - T: Serialize, -{ - match optional { - None => serializer.serialize_str("None"), - Some(value) => T::serialize(value, serializer) - } + config: Option, } /// Configuration for the server. Each field will be read from environment variables. @@ -229,6 +205,18 @@ impl TicketServerConfig { pub fn environment(&self) -> Option<&str> { self.service_info.environment() } + + pub fn set_ticket_server_addr(&mut self, ticket_server_addr: SocketAddr) { + self.ticket_server_addr = ticket_server_addr; + } + + pub fn set_cors(&mut self, cors: CorsConfig) { + self.cors = cors; + } + + pub fn set_service_info(&mut self, service_info: ServiceInfo) { + self.service_info = service_info; + } } /// Configuration for the htsget server. @@ -236,7 +224,7 @@ impl TicketServerConfig { #[serde(default)] pub struct DataServerConfig { addr: SocketAddr, - path: PathBuf, + local_path: PathBuf, serve_at: PathBuf, key: Option, cert: Option, @@ -249,8 +237,8 @@ impl DataServerConfig { self.addr } - pub fn path(&self) -> &Path { - &self.path + pub fn local_path(&self) -> &Path { + &self.local_path } pub fn serve_at(&self) -> &Path { @@ -297,8 +285,8 @@ impl DataServerConfig { self.addr = addr; } - pub fn set_path(&mut self, path: PathBuf) { - self.path = path; + pub fn set_local_path(&mut self, path: PathBuf) { + self.local_path = path; } pub fn set_serve_at(&mut self, serve_at: PathBuf) { @@ -324,7 +312,7 @@ impl Default for DataServerConfig { addr: default_localstorage_addr() .parse() .expect("expected valid address"), - path: default_path().into(), + local_path: default_path().into(), serve_at: default_serve_at().into(), key: None, cert: None, @@ -406,7 +394,7 @@ impl Default for Config { Self { ticket_server: TicketServerConfig::default(), data_server: DataServerConfigOption::Some(DataServerConfig::default()), - resolvers: vec![RegexResolver::default(), RegexResolver::default()], + resolvers: vec![RegexResolver::default()], } } } @@ -414,7 +402,7 @@ impl Default for Config { impl Config { /// Parse the command line arguments pub fn parse_args() -> PathBuf { - Args::parse().config + Args::parse().config.unwrap_or_else(|| "".into()) } /// Read the environment variables into a Config struct. @@ -688,21 +676,22 @@ mod tests { } #[test] - fn config_storage_type_url_file() { + fn config_storage_type_local_file() { test_config_from_file( r#" [[resolvers]] regex = "regex" [resolvers.storage_type] - type = "Url" - path = "path" + type = "Local" + local_path = "path" scheme = "HTTPS" + path_prefix = "path" "#, |config| { assert!(matches!( config.resolvers().first().unwrap().storage_type(), - StorageType::Url(resolver) if resolver.path() == "path" && resolver.scheme() == Scheme::Https + StorageType::Local(resolver) if resolver.local_path() == "path" && resolver.scheme() == Scheme::Https && resolver.path_prefix() == "path" )); }, ); diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index d0610e771..6a6bea5d3 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -221,6 +221,11 @@ impl Query { } } + pub fn with_id(mut self, id: impl Into) -> Self { + self.id = id.into(); + self + } + pub fn with_format(mut self, format: Format) -> Self { self.format = format; self diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index b1a6178ce..bf05fa51d 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -3,7 +3,7 @@ use regex::{Error, Regex}; use serde::{Deserialize, Serialize}; use tracing::instrument; -use crate::config::{default_localstorage_addr, default_serve_at}; +use crate::config::{default_localstorage_addr, default_path, default_serve_at}; use crate::regex_resolver::aws::S3Resolver; use crate::Format::{Bam, Bcf, Cram, Vcf}; use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; @@ -30,7 +30,7 @@ pub trait QueryMatcher { #[non_exhaustive] pub enum StorageType { #[serde(alias = "url", alias = "URL")] - Url(UrlResolver), + Local(LocalResolver), #[cfg(feature = "s3-storage")] #[serde(alias = "s3")] S3(S3Resolver), @@ -38,7 +38,7 @@ pub enum StorageType { impl Default for StorageType { fn default() -> Self { - Self::Url(UrlResolver::default()) + Self::Local(LocalResolver::default()) } } @@ -59,14 +59,15 @@ impl Default for Scheme { /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] -pub struct UrlResolver { +pub struct LocalResolver { scheme: Scheme, #[serde(with = "http_serde::authority")] authority: Authority, - path: String, + local_path: String, + path_prefix: String } -impl UrlResolver { +impl LocalResolver { pub fn scheme(&self) -> Scheme { self.scheme } @@ -75,17 +76,38 @@ impl UrlResolver { &self.authority } - pub fn path(&self) -> &str { - &self.path + pub fn local_path(&self) -> &str { + &self.local_path + } + + pub fn path_prefix(&self) -> &str { + &self.path_prefix + } + + pub fn set_scheme(&mut self, scheme: Scheme) { + self.scheme = scheme; + } + + pub fn set_authority(&mut self, authority: Authority) { + self.authority = authority; + } + + pub fn set_local_path(&mut self, local_path: String) { + self.local_path = local_path; + } + + pub fn set_path_prefix(&mut self, path_prefix: String) { + self.path_prefix = path_prefix; } } -impl Default for UrlResolver { +impl Default for LocalResolver { fn default() -> Self { Self { scheme: Scheme::default(), authority: Authority::from_static(default_localstorage_addr()), - path: default_serve_at().to_string(), + local_path: default_path().into(), + path_prefix: default_serve_at().into(), } } } @@ -288,15 +310,6 @@ impl Resolver for RegexResolver { } } -// impl<'a, I> Resolver for I -// where -// I: Iterator, -// { -// fn resolve_id(&self, query: &Query) -> Option { -// self.find_map(|resolver| resolver.resolve_id(query)) -// } -// } - #[cfg(test)] pub mod tests { use super::*; diff --git a/htsget-http-actix/src/main.rs b/htsget-http-actix/src/main.rs index 7d2c3b996..9b83b3d66 100644 --- a/htsget-http-actix/src/main.rs +++ b/htsget-http-actix/src/main.rs @@ -18,7 +18,7 @@ async fn main() -> std::io::Result<()> { let server = server.clone(); let mut formatter = HttpTicketFormatter::try_from(server.clone())?; let local_server = formatter.bind_data_server().await?; - let local_server = tokio::spawn(async move { local_server.serve(&server.path()).await }); + let local_server = tokio::spawn(async move { local_server.serve(&server.local_path()).await }); let ticket_server_config = config.ticket_server().clone(); select! { diff --git a/htsget-http-core/src/query_builder.rs b/htsget-http-core/src/query_builder.rs index cc8c6b286..2742a472b 100644 --- a/htsget-http-core/src/query_builder.rs +++ b/htsget-http-core/src/query_builder.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use htsget_config::{Class, Fields, Format, Tags}; use tracing::instrument; @@ -164,7 +165,7 @@ impl QueryBuilder { }; if let Some(tags) = tags { - let tags: Vec = tags.into_iter().map(Into::into).collect(); + let tags: HashSet = tags.into_iter().map(Into::into).collect(); if tags.iter().any(|tag| notags.contains(tag)) { return Err(HtsGetError::InvalidInput( "tags and notags can't intersect".to_string(), @@ -319,11 +320,11 @@ mod tests { .with_fields(Some("header,part1,part2")) .build() .fields(), - &Fields::List(vec![ + &Fields::List(HashSet::from_iter(vec![ "header".to_string(), "part1".to_string(), "part2".to_string() - ]) + ])) ); } @@ -336,13 +337,13 @@ mod tests { .build(); assert_eq!( query.tags(), - &Tags::List(vec![ + &Tags::List(HashSet::from_iter(vec![ "header".to_string(), "part1".to_string(), "part2".to_string() - ]) + ])) ); - assert_eq!(query.no_tags(), &NoTags(Some(vec!["part3".to_string()]))); + assert_eq!(query.no_tags(), &NoTags(Some(HashSet::from_iter(vec!["part3".to_string()])))); } #[test] @@ -354,12 +355,12 @@ mod tests { .build(); assert_eq!( query.tags(), - &Tags::List(vec![ + &Tags::List(HashSet::from_iter(vec![ "header".to_string(), "part1".to_string(), "part2".to_string() - ]) + ])) ); - assert_eq!(query.no_tags(), &NoTags(Some(vec!["part3".to_string()]))); + assert_eq!(query.no_tags(), &NoTags(Some(HashSet::from_iter(vec!["part3".to_string()])))); } } diff --git a/htsget-http-lambda/src/lib.rs b/htsget-http-lambda/src/lib.rs index a617f1d8b..7ea630da1 100644 --- a/htsget-http-lambda/src/lib.rs +++ b/htsget-http-lambda/src/lib.rs @@ -205,7 +205,7 @@ mod tests { use std::sync::Arc; use async_trait::async_trait; - use htsget_config::regex_resolver::{RegexResolver, StorageType, UrlResolver}; + use htsget_config::regex_resolver::{RegexResolver, StorageType, LocalResolver}; use htsget_config::Class; use lambda_http::http::header::HeaderName; use lambda_http::http::Uri; diff --git a/htsget-search/src/htsget/from_storage.rs b/htsget-search/src/htsget/from_storage.rs index eb587b14b..14039bad9 100644 --- a/htsget-search/src/htsget/from_storage.rs +++ b/htsget-search/src/htsget/from_storage.rs @@ -46,16 +46,16 @@ impl HtsGet for &[RegexResolver] { for resolver in self.iter() { if let Some(id) = resolver.resolve_id(&query) { match resolver.storage_type() { - StorageType::Url(url) => { + StorageType::Local(url) => { let searcher = - HtsGetFromStorage::local_from(url.path(), resolver.clone(), url.clone())?; - return searcher.search(query).await; + HtsGetFromStorage::local_from(url.local_path(), resolver.clone(), url.clone())?; + return searcher.search(query.with_id(id)).await; } #[cfg(feature = "s3-storage")] StorageType::S3(s3) => { let searcher = HtsGetFromStorage::s3_from(s3.bucket().to_string(), resolver.clone()).await; - return searcher.search(query).await; + return searcher.search(query.with_id(id)).await; } _ => {} } diff --git a/htsget-search/src/htsget/mod.rs b/htsget-search/src/htsget/mod.rs index 2c8058ef3..0e18a8528 100644 --- a/htsget-search/src/htsget/mod.rs +++ b/htsget-search/src/htsget/mod.rs @@ -231,6 +231,7 @@ impl Response { #[cfg(test)] mod tests { use super::*; + use std::collections::HashSet; use htsget_config::{Fields, NoTags, Tags}; #[test] @@ -326,10 +327,10 @@ mod tests { #[test] fn query_with_fields() { let result = Query::new("NA12878", Format::Bam) - .with_fields(Fields::List(vec!["QNAME".to_string(), "FLAG".to_string()])); + .with_fields(Fields::List(HashSet::from_iter(vec!["QNAME".to_string(), "FLAG".to_string()]))); assert_eq!( result.fields(), - &Fields::List(vec!["QNAME".to_string(), "FLAG".to_string()]) + &Fields::List(HashSet::from_iter(vec!["QNAME".to_string(), "FLAG".to_string()])) ); } @@ -344,7 +345,7 @@ mod tests { let result = Query::new("NA12878", Format::Bam).with_no_tags(vec!["RG", "OQ"]); assert_eq!( result.no_tags(), - &NoTags(Some(vec!["RG".to_string(), "OQ".to_string()])) + &NoTags(Some(HashSet::from_iter(vec!["RG".to_string(), "OQ".to_string()]))) ); } diff --git a/htsget-search/src/htsget/search.rs b/htsget-search/src/htsget/search.rs index b9d19e944..ff2b84752 100644 --- a/htsget-search/src/htsget/search.rs +++ b/htsget-search/src/htsget/search.rs @@ -205,7 +205,7 @@ where /// Get the position at the end of file marker. #[instrument(level = "trace", skip(self), ret)] async fn position_at_eof(&self, query: &Query) -> Result { - let file_size = self.get_storage().head(query).await?; + let file_size = self.get_storage().head(query.format().fmt_file(query.id())).await?; Ok( file_size - u64::try_from(self.get_eof_marker().len()) @@ -217,7 +217,7 @@ where #[instrument(level = "trace", skip(self))] async fn read_index(&self, query: &Query) -> Result { trace!("reading index"); - let storage = self.get_storage().get(query, GetOptions::default()).await?; + let storage = self.get_storage().get(query.format().fmt_index(query.id()), GetOptions::default()).await?; Self::read_index_inner(storage) .await .map_err(|err| HtsGetError::io_error(format!("reading {} index: {}", self.get_format(), err))) @@ -289,7 +289,7 @@ where let query_owned = query.clone(); storage_futures.push_back(tokio::spawn(async move { storage - .range_url(&query_owned, RangeUrlOptions::from(range)) + .range_url(query_owned.format().fmt_file(query_owned.id()), RangeUrlOptions::from(range)) .await })); } @@ -314,7 +314,7 @@ where trace!("getting header"); let get_options = GetOptions::default().with_range(self.get_byte_ranges_for_header(index).await?); - let reader_type = self.get_storage().get(query, get_options).await?; + let reader_type = self.get_storage().get(query.format().fmt_file(query.id()), get_options).await?; let mut reader = Self::init_reader(reader_type); Self::read_raw_header(&mut reader) @@ -408,7 +408,7 @@ where Ok(chunks) }); - let gzi_data = self.get_storage().get(query, GetOptions::default()).await; + let gzi_data = self.get_storage().get(query.format().fmt_gzi(query.id())?, GetOptions::default()).await; let byte_ranges: Vec = match gzi_data { Ok(gzi_data) => { let span = trace_span!("reading gzi"); diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index afc044e6d..7cc948c91 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -2,7 +2,7 @@ pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketSer #[cfg(feature = "s3-storage")] pub use htsget_config::regex_resolver::aws::S3Resolver; pub use htsget_config::regex_resolver::{ - QueryMatcher, RegexResolver, Resolver, StorageType, UrlResolver, + QueryMatcher, RegexResolver, Resolver, StorageType, LocalResolver, }; pub mod htsget; diff --git a/htsget-search/src/storage/aws.rs b/htsget-search/src/storage/aws.rs index c148b4b38..fd2721f32 100644 --- a/htsget-search/src/storage/aws.rs +++ b/htsget-search/src/storage/aws.rs @@ -22,7 +22,7 @@ use htsget_config::Query; use crate::htsget::Url; use crate::storage::aws::Retrieval::{Delayed, Immediate}; use crate::storage::StorageError::AwsS3Error; -use crate::storage::{resolve_id, BytesPosition, StorageError}; +use crate::storage::{BytesPosition, StorageError}; use crate::storage::{BytesRange, Storage}; use crate::RegexResolver; @@ -65,41 +65,45 @@ impl AwsS3Storage { ) } - pub async fn s3_presign_url(&self, query: &Query, range: BytesPosition) -> Result { + pub async fn s3_presign_url + Send>( + &self, + key: K, + range: BytesPosition, + ) -> Result { let response = self .client .get_object() .bucket(&self.bucket) - .key(resolve_id(&self.id_resolver, query)?); + .key(key.as_ref()); let response = Self::apply_range(response, range); Ok( response .presigned( PresigningConfig::expires_in(Duration::from_secs(Self::PRESIGNED_REQUEST_EXPIRY)) - .map_err(|err| AwsS3Error(err.to_string(), query.id().to_string()))?, + .map_err(|err| AwsS3Error(err.to_string(), key.as_ref().to_string()))?, ) .await - .map_err(|err| AwsS3Error(err.to_string(), query.id().to_string()))? + .map_err(|err| AwsS3Error(err.to_string(), key.as_ref().to_string()))? .uri() .to_string(), ) } - async fn s3_head(&self, query: &Query) -> Result { + async fn s3_head + Send>(&self, key: K) -> Result { self .client .head_object() .bucket(&self.bucket) - .key(resolve_id(&self.id_resolver, query)?) + .key(key.as_ref()) .send() .await - .map_err(|err| AwsS3Error(err.to_string(), query.id().to_string())) + .map_err(|err| AwsS3Error(err.to_string(), key.as_ref().to_string())) } /// Returns the retrieval type of the object stored with the key. #[instrument(level = "trace", skip_all, ret)] - pub async fn get_retrieval_type(&self, query: &Query) -> Result { - let head = self.s3_head(query).await?; + pub async fn get_retrieval_type + Send>(&self, key: K) -> Result { + let head = self.s3_head(key.as_ref()).await?; Ok( // Default is Standard. match head.storage_class.unwrap_or(StorageClass::Standard) { @@ -138,11 +142,15 @@ impl AwsS3Storage { } } - pub async fn get_content(&self, query: &Query, options: GetOptions) -> Result { - if let Delayed(class) = self.get_retrieval_type(query).await? { + pub async fn get_content + Send>( + &self, + key: K, + options: GetOptions, + ) -> Result { + if let Delayed(class) = self.get_retrieval_type(key.as_ref()).await? { return Err(AwsS3Error( format!("cannot retrieve object immediately, class is `{:?}`", class), - query.id().to_string(), + key.as_ref().to_string(), )); } @@ -150,23 +158,23 @@ impl AwsS3Storage { .client .get_object() .bucket(&self.bucket) - .key(resolve_id(&self.id_resolver, query)?); + .key(key.as_ref()); let response = Self::apply_range(response, options.range); Ok( response .send() .await - .map_err(|err| AwsS3Error(err.to_string(), query.id().to_string()))? + .map_err(|err| AwsS3Error(err.to_string(), key.as_ref().to_string()))? .body, ) } - async fn create_stream_reader( + async fn create_stream_reader + Send>( &self, - query: &Query, + key: K, options: GetOptions, ) -> Result> { - let response = self.get_content(query, options).await?; + let response = self.get_content(key, options).await?; Ok(StreamReader::new(response)) } } @@ -177,26 +185,38 @@ impl Storage for AwsS3Storage { /// Gets the actual s3 object as a buffered reader. #[instrument(level = "trace", skip(self))] - async fn get(&self, query: &Query, options: GetOptions) -> Result { - debug!(calling_from = ?self, id = query.id(), "getting file with key {:?}", query.id()); + async fn get + Send + Debug>( + &self, + key: K, + options: GetOptions, + ) -> Result { + let key = key.as_ref(); + debug!(calling_from = ?self, key, "getting file with key {:?}", key); - self.create_stream_reader(query, options).await + self.create_stream_reader(key, options).await } /// Returns a S3-presigned htsget URL #[instrument(level = "trace", skip(self))] - async fn range_url(&self, query: &Query, options: RangeUrlOptions) -> Result { - let presigned_url = self.s3_presign_url(query, options.range.clone()).await?; + async fn range_url + Send + Debug>( + &self, + key: K, + options: RangeUrlOptions, + ) -> Result { + let key = key.as_ref(); + let presigned_url = self.s3_presign_url(key, options.range.clone()).await?; let url = options.apply(Url::new(presigned_url)); - debug!(calling_from = ?self, id = query.id(), ?url, "getting url with key {:?}", query.id()); + debug!(calling_from = ?self, key, ?url, "getting url with key {:?}", key); Ok(url) } /// Returns the size of the S3 object in bytes. #[instrument(level = "trace", skip(self))] - async fn head(&self, query: &Query) -> Result { - let head = self.s3_head(query).await?; + async fn head + Send + Debug>(&self, key: K) -> Result { + let key = key.as_ref(); + + let head = self.s3_head(key).await?; let len = u64::try_from(head.content_length).map_err(|err| { StorageError::IoError( "failed to convert file length to `u64`".to_string(), @@ -204,7 +224,7 @@ impl Storage for AwsS3Storage { ) })?; - debug!(calling_from = ?self, id = query.id(), len, "size of key {:?} is {}", query.id(), len); + debug!(calling_from = ?self, key, len, "size of key {:?} is {}", key, len); Ok(len) } } @@ -225,7 +245,7 @@ mod tests { use s3_server::storages::fs::FileSystem; use s3_server::{S3Service, SimpleAuth}; - use htsget_config::regex_resolver::UrlResolver; + use htsget_config::regex_resolver::LocalResolver; use htsget_config::Format::Bam; use htsget_config::Query; @@ -293,7 +313,7 @@ mod tests { async fn existing_key() { with_aws_s3_storage(|storage| async move { let result = storage - .get(&Query::new("key2", Bam), GetOptions::default()) + .get("key2", GetOptions::default()) .await; assert!(matches!(result, Ok(_))); }) @@ -304,7 +324,7 @@ mod tests { async fn non_existing_key() { with_aws_s3_storage(|storage| async move { let result = storage - .get(&Query::new("non-existing-key", Bam), GetOptions::default()) + .get("non-existing-key", GetOptions::default()) .await; assert!(matches!(result, Err(StorageError::AwsS3Error(_, _)))); }) @@ -316,7 +336,7 @@ mod tests { with_aws_s3_storage(|storage| async move { let result = storage .range_url( - &Query::new("non-existing-key", Bam), + "non-existing-key", RangeUrlOptions::default(), ) .await; @@ -329,7 +349,7 @@ mod tests { async fn url_of_existing_key() { with_aws_s3_storage(|storage| async move { let result = storage - .range_url(&Query::new("key2", Bam), RangeUrlOptions::default()) + .range_url("key2", RangeUrlOptions::default()) .await .unwrap(); assert!(result @@ -348,7 +368,7 @@ mod tests { with_aws_s3_storage(|storage| async move { let result = storage .range_url( - &Query::new("key2", Bam), + "key2", RangeUrlOptions::default().with_range(BytesPosition::new(Some(7), Some(9), None)), ) .await @@ -374,7 +394,7 @@ mod tests { with_aws_s3_storage(|storage| async move { let result = storage .range_url( - &Query::new("key2", Bam), + "key2", RangeUrlOptions::default().with_range(BytesPosition::new(Some(7), None, None)), ) .await @@ -398,7 +418,7 @@ mod tests { #[tokio::test] async fn file_size() { with_aws_s3_storage(|storage| async move { - let result = storage.head(&Query::new("key2", Bam)).await; + let result = storage.head("key2").await; let expected: u64 = 6; assert!(matches!(result, Ok(size) if size == expected)); }) @@ -408,7 +428,7 @@ mod tests { #[tokio::test] async fn retrieval_type() { with_aws_s3_storage(|storage| async move { - let result = storage.get_retrieval_type(&Query::new("key2", Bam)).await; + let result = storage.get_retrieval_type("key2").await; println!("{:?}", result); }) .await; diff --git a/htsget-search/src/storage/data_server.rs b/htsget-search/src/storage/data_server.rs index 7ee5c9678..b966554af 100644 --- a/htsget-search/src/storage/data_server.rs +++ b/htsget-search/src/storage/data_server.rs @@ -17,7 +17,7 @@ use axum_extra::routing::SpaRouter; use futures_util::future::poll_fn; use htsget_config::config::cors::CorsConfig; use htsget_config::config::DataServerConfig; -use htsget_config::regex_resolver::UrlResolver; +use htsget_config::regex_resolver::LocalResolver; use http::uri::Scheme; use hyper::server::accept::Accept; use hyper::server::conn::{AddrIncoming, Http}; @@ -271,7 +271,7 @@ mod tests { test_cors_preflight_request_uri, test_cors_simple_request_uri, }; use htsget_test_utils::http_tests::{ - default_test_config, Header, Response as TestResponse, TestRequest, TestServer, + default_test_config, default_cors_config, Header, Response as TestResponse, TestRequest, TestServer, }; use htsget_test_utils::util::generate_test_certificates; @@ -469,7 +469,7 @@ mod tests { P: AsRef + Send + 'static, { let addr = SocketAddr::from_str(&format!("{}:{}", "127.0.0.1", "0")).unwrap(); - let server = DataServer::bind_addr(addr, "/data", cert_key_pair, CorsConfig::default()) + let server = DataServer::bind_addr(addr, "/data", cert_key_pair, default_cors_config()) .await .unwrap(); let port = server.local_addr().port(); diff --git a/htsget-search/src/storage/local.rs b/htsget-search/src/storage/local.rs index 6bab3a15c..014ea9a90 100644 --- a/htsget-search/src/storage/local.rs +++ b/htsget-search/src/storage/local.rs @@ -36,7 +36,7 @@ impl LocalStorage { .as_ref() .to_path_buf() .canonicalize() - .map_err(|_| StorageError::KeyNotFound(base_path.as_ref().to_string_lossy().to_string())) + .map_err(|_| StorageError::KeyNotFound(base_path.as_ref().to_string_lossy().to_string())) .map(|canonicalized_base_path| Self { base_path: canonicalized_base_path, id_resolver, @@ -48,31 +48,32 @@ impl LocalStorage { self.base_path.as_path() } - pub(crate) fn get_path_from_key(&self, query: &Query) -> Result { + pub(crate) fn get_path_from_key>(&self, key: K) -> Result { + let key: &str = key.as_ref(); self .base_path - .join(resolve_id(&self.id_resolver, query)?) + .join(key) .canonicalize() - .map_err(|_| StorageError::InvalidKey(query.id().to_string())) + .map_err(|_| StorageError::InvalidKey(key.to_string())) .and_then(|path| { path .starts_with(&self.base_path) .then_some(path) - .ok_or_else(|| StorageError::InvalidKey(query.id().to_string())) + .ok_or_else(|| StorageError::InvalidKey(key.to_string())) }) .and_then(|path| { path .is_file() .then_some(path) - .ok_or_else(|| StorageError::KeyNotFound(query.id().to_string())) + .ok_or_else(|| StorageError::KeyNotFound(key.to_string())) }) } - pub async fn get(&self, query: &Query) -> Result { - let path = self.get_path_from_key(query)?; + pub async fn get>(&self, key: K) -> Result { + let path = self.get_path_from_key(&key)?; File::open(path) .await - .map_err(|_| StorageError::KeyNotFound(query.id().to_string())) + .map_err(|_| StorageError::KeyNotFound(key.as_ref().to_string())) } } @@ -82,15 +83,19 @@ impl Storage for LocalStorage { /// Get the file at the location of the key. #[instrument(level = "debug", skip(self))] - async fn get(&self, query: &Query, _options: GetOptions) -> Result { - debug!(calling_from = ?self, id = query.id(), "getting file with key {:?}", query.id()); - self.get(query).await + async fn get + Send + Debug>(&self, key: K, _options: GetOptions) -> Result { + debug!(calling_from = ?self, key = key.as_ref(), "getting file with key {:?}", key.as_ref()); + self.get(key).await } /// Get a url for the file at key. #[instrument(level = "debug", skip(self))] - async fn range_url(&self, query: &Query, options: RangeUrlOptions) -> Result { - let path = self.get_path_from_key(query)?; + async fn range_url + Send + Debug>( + &self, + key: K, + options: RangeUrlOptions, + ) -> Result { + let path = self.get_path_from_key(&key)?; let path = path .strip_prefix(&self.base_path) .map_err(|err| StorageError::InternalError(err.to_string()))? @@ -99,20 +104,20 @@ impl Storage for LocalStorage { let url = Url::new(self.url_formatter.format_url(&path)?); let url = options.apply(url); - debug!(calling_from = ?self, id = query.id(), ?url, "getting url with key {:?}", query.id()); + debug!(calling_from = ?self, key = key.as_ref(), ?url, "getting url with key {:?}", key.as_ref()); Ok(url) } /// Get the size of the file. #[instrument(level = "debug", skip(self))] - async fn head(&self, query: &Query) -> Result { - let path = self.get_path_from_key(query)?; + async fn head + Send + Debug>(&self, key: K) -> Result { + let path = self.get_path_from_key(&key)?; let len = tokio::fs::metadata(path) .await .map_err(|err| StorageError::KeyNotFound(err.to_string()))? .len(); - debug!(calling_from = ?self, id = query.id(), len, "size of key {:?} is {}", query.id(), len); + debug!(calling_from = ?self, key = key.as_ref(), len, "size of key {:?} is {}", key.as_ref(), len); Ok(len) } } @@ -139,7 +144,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_non_existing_key() { with_local_storage(|storage| async move { - let result = storage.get(&Query::new("non-existing-key", Bam)).await; + let result = storage.get("non-existing-key").await; assert!(matches!(result, Err(StorageError::InvalidKey(msg)) if msg == "non-existing-key")); }) .await; @@ -148,7 +153,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_folder() { with_local_storage(|storage| async move { - let result = Storage::get(&storage, &Query::new("folder", Bam), GetOptions::default()).await; + let result = Storage::get(&storage, "folder", GetOptions::default()).await; assert!(matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "folder")); }) .await; @@ -159,7 +164,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::get( &storage, - &Query::new("folder/../../passwords", Bam), + "folder/../../passwords", GetOptions::default(), ) .await; @@ -175,7 +180,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::get( &storage, - &Query::new("folder/../key1", Bam), + "folder/../key1", GetOptions::default(), ) .await; @@ -189,7 +194,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::range_url( &storage, - &Query::new("non-existing-key", Bam), + "non-existing-key", RangeUrlOptions::default(), ) .await; @@ -203,7 +208,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::range_url( &storage, - &Query::new("folder", Bam), + "folder", RangeUrlOptions::default(), ) .await; @@ -217,7 +222,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::range_url( &storage, - &Query::new("folder/../../passwords", Bam), + "folder/../../passwords", RangeUrlOptions::default(), ) .await; @@ -233,7 +238,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::range_url( &storage, - &Query::new("folder/../key1", Bam), + "folder/../key1", RangeUrlOptions::default(), ) .await; @@ -248,7 +253,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::range_url( &storage, - &Query::new("folder/../key1", Bam), + "folder/../key1", RangeUrlOptions::default().with_range(BytesPosition::new(Some(7), Some(10), None)), ) .await; @@ -264,7 +269,7 @@ pub(crate) mod tests { with_local_storage(|storage| async move { let result = Storage::range_url( &storage, - &Query::new("folder/../key1", Bam), + "folder/../key1", RangeUrlOptions::default().with_range(BytesPosition::new(Some(7), None, None)), ) .await; @@ -278,7 +283,7 @@ pub(crate) mod tests { #[tokio::test] async fn file_size() { with_local_storage(|storage| async move { - let result = Storage::head(&storage, &Query::new("folder/../key1", Bam)).await; + let result = Storage::head(&storage, "folder/../key1").await; let expected: u64 = 6; assert!(matches!(result, Ok(size) if size == expected)); }) diff --git a/htsget-search/src/storage/mod.rs b/htsget-search/src/storage/mod.rs index 51f377d78..6fe088ad9 100644 --- a/htsget-search/src/storage/mod.rs +++ b/htsget-search/src/storage/mod.rs @@ -10,7 +10,7 @@ use std::time::Duration; use async_trait::async_trait; use base64::encode; use htsget_config::config::cors::{AllowType, CorsConfig, TaggedAllowTypes, TaggedAnyAllowType}; -use htsget_config::regex_resolver::{Scheme, UrlResolver}; +use htsget_config::regex_resolver::{Scheme, LocalResolver}; use htsget_config::{Class, Query}; use http::{uri, HeaderValue, Method}; use thiserror::Error; @@ -37,13 +37,21 @@ pub trait Storage { type Streamable: AsyncRead + Unpin + Send; /// Get the object using the key. - async fn get(&self, query: &Query, options: GetOptions) -> Result; + async fn get + Send + Debug>( + &self, + key: K, + options: GetOptions, + ) -> Result; /// Get the url of the object represented by the key using a bytes range. - async fn range_url(&self, query: &Query, options: RangeUrlOptions) -> Result; + async fn range_url + Send + Debug>( + &self, + key: K, + options: RangeUrlOptions, + ) -> Result; /// Get the size of the object represented by the key. - async fn head(&self, query: &Query) -> Result; + async fn head + Send + Debug>(&self, key: K) -> Result; /// Get the url of the object using an inline data uri. #[instrument(level = "trace", ret)] @@ -92,7 +100,7 @@ pub enum StorageError { AwsS3Error(String, String), } -impl UrlFormatter for UrlResolver { +impl UrlFormatter for LocalResolver { fn format_url>(&self, key: K) -> Result { uri::Builder::new() .scheme(match self.scheme() { @@ -100,7 +108,7 @@ impl UrlFormatter for UrlResolver { Scheme::Https => uri::Scheme::HTTPS, }) .authority(self.authority().to_string()) - .path_and_query(format!("{}/{}", self.path(), key.as_ref())) + .path_and_query(format!("{}/{}", self.path_prefix(), key.as_ref())) .build() .map_err(|err| StorageError::InvalidUri(err.to_string())) .map(|value| value.to_string()) diff --git a/htsget-test-utils/src/cors_tests.rs b/htsget-test-utils/src/cors_tests.rs index 4b607e5ee..78a5df0e8 100644 --- a/htsget-test-utils/src/cors_tests.rs +++ b/htsget-test-utils/src/cors_tests.rs @@ -70,7 +70,7 @@ pub async fn test_cors_preflight_request_uri( .get(ACCESS_CONTROL_ALLOW_ORIGIN) .unwrap() .to_str() - .unwrap(), + .unwrap().to_lowercase(), "http://example.com" ); @@ -80,19 +80,17 @@ pub async fn test_cors_preflight_request_uri( .get(ACCESS_CONTROL_ALLOW_HEADERS) .unwrap() .to_str() - .unwrap(), - "X-Requested-With" + .unwrap() + .to_lowercase(), + "x-requested-with" ); - for method in &[ - "HEAD", "GET", "OPTIONS", "PUT", "PATCH", "TRACE", "POST", "DELETE", "CONNECT", - ] { - assert!(response - .headers - .get(ACCESS_CONTROL_ALLOW_METHODS) - .unwrap() - .to_str() - .unwrap() - .contains(method)); - } + assert!( + response + .headers + .get(ACCESS_CONTROL_ALLOW_METHODS) + .unwrap() + .to_str() + .unwrap().to_lowercase().contains("post") + ); } diff --git a/htsget-test-utils/src/http_tests.rs b/htsget-test-utils/src/http_tests.rs index 78e4e8f93..d89322dd0 100644 --- a/htsget-test-utils/src/http_tests.rs +++ b/htsget-test-utils/src/http_tests.rs @@ -1,11 +1,14 @@ use std::fs; +use std::net::{SocketAddr, TcpListener}; use std::path::{Path, PathBuf}; +use std::str::FromStr; use async_trait::async_trait; use htsget_config::config::cors::{AllowType, CorsConfig}; -use htsget_config::config::DataServerConfig; -use htsget_config::regex_resolver::RegexResolver; +use htsget_config::config::{DataServerConfig, TicketServerConfig}; +use htsget_config::regex_resolver::{LocalResolver, RegexResolver, Scheme, StorageType}; use http::HeaderMap; +use http::uri::Authority; use serde::de; use crate::util::generate_test_certificates; @@ -87,12 +90,27 @@ pub fn default_dir_data() -> PathBuf { } fn set_path(config: &mut DataServerConfig) { - config.set_path(default_dir_data()); + config.set_local_path(default_dir_data()); } -fn set_addr_and_path(config: &mut DataServerConfig) { +fn set_addr_and_path(config: &mut DataServerConfig, addr: SocketAddr) { set_path(config); - config.set_addr("127.0.0.1:0".parse().unwrap()); + config.set_addr(addr); +} + +/// Get the default test resolver. +pub fn default_test_resolver(addr: SocketAddr, scheme: Scheme) -> RegexResolver { + let mut resolver = LocalResolver::default(); + resolver.set_local_path(default_dir_data().to_str().unwrap().to_string()); + resolver.set_authority(Authority::from_str(&addr.to_string()).unwrap()); + resolver.set_scheme(scheme); + + RegexResolver::new( + StorageType::Local(resolver), + ".*", + "$0", + Default::default() + ).unwrap() } /// Default config with fixed port. @@ -100,28 +118,48 @@ pub fn default_config_fixed_port() -> Config { let mut config = Config::default(); let mut data_server_config = DataServerConfig::default(); + let addr = data_server_config.addr(); set_path(&mut data_server_config); config.set_data_server(Some(data_server_config)); + config.set_resolvers(vec![default_test_resolver(addr, Scheme::Http)]); + config } -/// Default config using the current cargo manifest directory, and dynamic port. -pub fn default_test_config() -> Config { - let mut server_config = DataServerConfig::default(); - set_addr_and_path(&mut server_config); +fn get_dynamic_addr() -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap() +} - let mut server_config = DataServerConfig::default(); +/// Set the default cors testing config. +pub fn default_cors_config() -> CorsConfig { let mut cors = CorsConfig::default(); cors.set_allow_credentials(false); cors.set_allow_origins(AllowType::List(vec!["http://example.com".parse().unwrap()])); - server_config.set_cors(cors); + cors +} + +/// Default config using the current cargo manifest directory, and dynamic port. +pub fn default_test_config() -> Config { + let mut server_config = DataServerConfig::default(); + let addr = get_dynamic_addr(); + + set_addr_and_path(&mut server_config, addr); + + let mut cors = default_cors_config(); + server_config.set_cors(cors.clone()); let mut config = Config::default(); + let mut ticket_server_config = TicketServerConfig::default(); + ticket_server_config.set_cors(cors); + + config.set_ticket_server(ticket_server_config); config.set_data_server(Some(server_config)); + config.set_resolvers(vec![default_test_resolver(addr, Scheme::Http)]); config } @@ -129,17 +167,21 @@ pub fn default_test_config() -> Config { /// Config with tls ticket server, using the current cargo manifest directory. pub fn config_with_tls>(path: P) -> Config { let mut server_config = DataServerConfig::default(); - set_addr_and_path(&mut server_config); + let addr = get_dynamic_addr(); + + set_addr_and_path(&mut server_config, addr); let (key_path, cert_path) = generate_test_certificates(path, "key.pem", "cert.pem"); - let mut server_config = DataServerConfig::default(); server_config.set_key(Some(key_path)); server_config.set_cert(Some(cert_path)); let mut config = Config::default(); + config.set_data_server(Some(server_config)); + config.set_resolvers(vec![default_test_resolver(addr, Scheme::Https)]); + config } diff --git a/htsget-test-utils/src/server_tests.rs b/htsget-test-utils/src/server_tests.rs index ed3770b77..a08de80af 100644 --- a/htsget-test-utils/src/server_tests.rs +++ b/htsget-test-utils/src/server_tests.rs @@ -1,14 +1,16 @@ use std::collections::HashMap; use std::path::PathBuf; +use std::time::Duration; use futures::future::join_all; use futures::TryStreamExt; -use htsget_config::regex_resolver::UrlResolver; +use htsget_config::regex_resolver::LocalResolver; use htsget_config::{Class, Format}; use http::Method; use noodles_bgzf as bgzf; use noodles_vcf as vcf; use reqwest::ClientBuilder; +use tokio::time::sleep; use htsget_http_core::{get_service_info_with, Endpoint}; use htsget_search::htsget::Response as HtsgetResponse; @@ -21,8 +23,11 @@ use crate::Config; /// Test response with with class. pub async fn test_response(response: Response, class: Class) { + println!("response: {:?}", response); assert!(response.is_success()); let body = response.deserialize_body::().unwrap(); + + println!("{:#?}", body); let expected_response = expected_response(class, response.expected_url_path); assert_eq!(body, expected_response); @@ -48,8 +53,7 @@ pub async fn test_response(response: Response, class: Class) { .unwrap(), ) .send() - .await - .unwrap() + .await.unwrap() .bytes() .await .unwrap() @@ -74,7 +78,9 @@ pub async fn test_response(response: Response, class: Class) { /// Create the a [HttpTicketFormatter], spawn the ticket server, returning the expected path and the formatter. pub async fn formatter_and_expected_path(config: &Config) -> (String, HttpTicketFormatter) { - let formatter = formatter_from_config(config).unwrap(); + let mut formatter = formatter_from_config(config).unwrap(); + spawn_ticket_server(config.data_server().unwrap().local_path().into(), &mut formatter).await; + (expected_url_path(&formatter), formatter) } From bbb33c021e7975fa002788547de75c8d9f2193fb Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 21 Dec 2022 08:25:30 +1100 Subject: [PATCH 29/45] refactor: reduce some options for cors, remove repeated code when configuring cors --- htsget-config/src/config/cors.rs | 79 ++++++++++++++++++++---- htsget-config/src/config/mod.rs | 44 +++++-------- htsget-config/src/lib.rs | 2 +- htsget-config/src/regex_resolver/mod.rs | 30 ++++----- htsget-http-actix/src/lib.rs | 63 ++++++++++--------- htsget-http-core/src/query_builder.rs | 12 +++- htsget-http-lambda/src/lib.rs | 2 +- htsget-search/src/htsget/mod.rs | 19 ++++-- htsget-search/src/htsget/search.rs | 25 ++++++-- htsget-search/src/lib.rs | 2 +- htsget-search/src/storage/aws.rs | 13 +--- htsget-search/src/storage/data_server.rs | 3 +- htsget-search/src/storage/local.rs | 38 +++--------- htsget-search/src/storage/mod.rs | 78 +++++++++++++---------- htsget-test-utils/src/cors_tests.rs | 21 ++++--- htsget-test-utils/src/http_tests.rs | 9 +-- htsget-test-utils/src/server_tests.rs | 9 ++- 17 files changed, 259 insertions(+), 190 deletions(-) diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs index 9325e5d06..fe70706fc 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/cors.rs @@ -40,6 +40,63 @@ pub enum AllowType { List(Vec), } +impl AllowType { + /// Apply a function to the builder when the type is a List. + pub fn apply_list(&self, func: F, builder: U) -> U + where + F: FnOnce(U, &Vec) -> U, + { + if let Self::List(list) = self { + func(builder, list) + } else { + builder + } + } + + /// Apply a function to the builder when the type is tagged. + pub fn apply_tagged(&self, func: F, builder: U, tagged_type: &Tagged) -> U + where + F: FnOnce(U) -> U, + Tagged: Eq, + { + if let Self::Tagged(tagged) = self { + if tagged == tagged_type { + return func(builder); + } + } + + builder + } +} + +impl AllowType { + /// Apply a function to the builder when the type is Mirror. + pub fn apply_mirror(&self, func: F, builder: U) -> U + where + F: FnOnce(U) -> U, + { + self.apply_tagged(func, builder, &TaggedAllowTypes::Mirror) + } + + /// Apply a function to the builder when the type is Any. + pub fn apply_any(&self, func: F, builder: U) -> U + where + F: FnOnce(U) -> U, + { + self.apply_tagged(func, builder, &TaggedAllowTypes::Any) + } +} + +impl AllowType { + /// Apply a function to the builder when the type is Any. + pub fn apply_any(&self, func: F, builder: U) -> U + where + F: FnOnce(U) -> U, + { + self.apply_tagged(func, builder, &TaggedAnyAllowType::Any) + } +} + fn serialize_allow_types(names: &Vec, serializer: S) -> Result where T: Display, @@ -94,8 +151,8 @@ impl Display for HeaderValue { pub struct CorsConfig { allow_credentials: bool, allow_origins: AllowType, - allow_headers: AllowType, - allow_methods: AllowType, + allow_headers: AllowType, + allow_methods: AllowType, max_age: usize, expose_headers: AllowType, } @@ -109,11 +166,11 @@ impl CorsConfig { &self.allow_origins } - pub fn allow_headers(&self) -> &AllowType { + pub fn allow_headers(&self) -> &AllowType { &self.allow_headers } - pub fn allow_methods(&self) -> &AllowType { + pub fn allow_methods(&self) -> &AllowType { &self.allow_methods } @@ -133,11 +190,11 @@ impl CorsConfig { self.allow_origins = allow_origins; } - pub fn set_allow_headers(&mut self, allow_headers: AllowType) { + pub fn set_allow_headers(&mut self, allow_headers: AllowType) { self.allow_headers = allow_headers; } - pub fn set_allow_methods(&mut self, allow_methods: AllowType) { + pub fn set_allow_methods(&mut self, allow_methods: AllowType) { self.allow_methods = allow_methods; } @@ -157,8 +214,8 @@ impl Default for CorsConfig { allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static( default_server_origin(), ))]), - allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), - allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), + allow_headers: AllowType::Tagged(TaggedAnyAllowType::Any), + allow_methods: AllowType::Tagged(TaggedAnyAllowType::Any), max_age: CORS_MAX_AGE, expose_headers: AllowType::List(vec![]), } @@ -189,7 +246,7 @@ mod tests { fn unit_variant_any_allow_type() { test_cors_config( "allow_methods = \"Any\"", - &AllowType::Tagged(TaggedAllowTypes::Any), + &AllowType::Tagged(TaggedAnyAllowType::Any), |config| config.allow_methods(), ); } @@ -197,9 +254,9 @@ mod tests { #[test] fn unit_variant_mirror_allow_type() { test_cors_config( - "allow_methods = \"Mirror\"", + "allow_origins = \"Mirror\"", &AllowType::Tagged(TaggedAllowTypes::Mirror), - |config| config.allow_methods(), + |config| config.allow_origins(), ); } diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index d7e97eb63..39ec79c1f 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -10,8 +10,8 @@ use std::path::{Path, PathBuf}; use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAnyAllowType}; use clap::Parser; use figment::providers::{Env, Format, Serialized, Toml}; -use figment::Figment; use figment::value::Value::Dict; +use figment::Figment; use http::header::HeaderName; use http::Method; use serde::de::IntoDeserializer; @@ -105,7 +105,7 @@ pub struct Config { #[derive(Serialize, Deserialize, Debug, Clone)] enum DataServerConfigNone { #[serde(alias = "none", alias = "NONE", alias = "")] - None + None, } /// Data server config enum options. @@ -113,7 +113,7 @@ enum DataServerConfigNone { #[serde(untagged)] enum DataServerConfigOption { None(DataServerConfigNone), - Some(DataServerConfig) + Some(DataServerConfig), } with_prefix!(ticket_server_prefix "ticket_server_"); @@ -150,11 +150,11 @@ impl TicketServerConfig { self.cors.allow_origins() } - pub fn allow_headers(&self) -> &AllowType { + pub fn allow_headers(&self) -> &AllowType { self.cors.allow_headers() } - pub fn allow_methods(&self) -> &AllowType { + pub fn allow_methods(&self) -> &AllowType { self.cors.allow_methods() } @@ -265,11 +265,11 @@ impl DataServerConfig { self.cors.allow_origins() } - pub fn allow_headers(&self) -> &AllowType { + pub fn allow_headers(&self) -> &AllowType { self.cors.allow_headers() } - pub fn allow_methods(&self) -> &AllowType { + pub fn allow_methods(&self) -> &AllowType { self.cors.allow_methods() } @@ -444,7 +444,7 @@ impl Config { pub fn data_server(&self) -> Option<&DataServerConfig> { match self.data_server { DataServerConfigOption::None(_) => None, - DataServerConfigOption::Some(ref config) => Some(config) + DataServerConfigOption::Some(ref config) => Some(config), } } @@ -464,8 +464,8 @@ impl Config { match data_server { None => { self.data_server = DataServerConfigOption::None(DataServerConfigNone::None); - }, - Some(value ) => { + } + Some(value) => { self.data_server = DataServerConfigOption::Some(value); } } @@ -566,15 +566,9 @@ mod tests { #[test] fn config_no_data_server_env() { - test_config_from_env( - vec![("HTSGET_DATA_SERVER", "")], - |config| { - assert!(matches!( - config.data_server(), - None - )); - }, - ); + test_config_from_env(vec![("HTSGET_DATA_SERVER", "")], |config| { + assert!(matches!(config.data_server(), None)); + }); } #[test] @@ -629,15 +623,9 @@ mod tests { #[test] fn config_no_data_server_file() { - test_config_from_file( - r#"data_server = """#, - |config| { - assert!(matches!( - config.data_server(), - None - )); - }, - ); + test_config_from_file(r#"data_server = """#, |config| { + assert!(matches!(config.data_server(), None)); + }); } #[test] diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 6a6bea5d3..385e38dae 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -3,10 +3,10 @@ extern crate core; use noodles::core::region::Interval as NoodlesInterval; use noodles::core::Position; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; use std::fmt::Formatter; use std::io::ErrorKind::Other; use std::{fmt, io}; -use std::collections::HashSet; use tracing::instrument; pub mod config; diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index bf05fa51d..f4036aeff 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -5,9 +5,9 @@ use tracing::instrument; use crate::config::{default_localstorage_addr, default_path, default_serve_at}; use crate::regex_resolver::aws::S3Resolver; +use crate::regex_resolver::ReferenceNames::All; use crate::Format::{Bam, Bcf, Cram, Vcf}; use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; -use crate::regex_resolver::ReferenceNames::All; #[cfg(feature = "s3-storage")] pub mod aws; @@ -64,7 +64,7 @@ pub struct LocalResolver { #[serde(with = "http_serde::authority")] authority: Authority, local_path: String, - path_prefix: String + path_prefix: String, } impl LocalResolver { @@ -142,7 +142,7 @@ pub struct QueryGuard { pub enum ReferenceNames { All, #[serde(with = "serde_regex")] - Some(Regex) + Some(Regex), } impl QueryGuard { @@ -198,7 +198,9 @@ impl QueryMatcher for Fields { fn query_matches(&self, query: &Query) -> bool { match (self, &query.fields) { (Fields::All, _) => true, - (Fields::List(self_fields), Fields::List(query_fields)) => self_fields.is_subset(query_fields), + (Fields::List(self_fields), Fields::List(query_fields)) => { + self_fields.is_subset(query_fields) + } (Fields::List(_), Fields::All) => false, } } @@ -217,16 +219,16 @@ impl QueryMatcher for Tags { impl QueryMatcher for QueryGuard { fn query_matches(&self, query: &Query) -> bool { self.allow_formats.contains(&query.format) - && self.allow_classes.contains(&query.class) - && self.allow_reference_names.query_matches(query) - && self - .allow_interval - .contains(query.interval.start.unwrap_or(u32::MIN)) - && self - .allow_interval - .contains(query.interval.end.unwrap_or(u32::MAX)) - && self.allow_fields.query_matches(query) - && self.allow_tags.query_matches(query) + && self.allow_classes.contains(&query.class) + && self.allow_reference_names.query_matches(query) + && self + .allow_interval + .contains(query.interval.start.unwrap_or(u32::MIN)) + && self + .allow_interval + .contains(query.interval.end.unwrap_or(u32::MAX)) + && self.allow_fields.query_matches(query) + && self.allow_tags.query_matches(query) } } diff --git a/htsget-http-actix/src/lib.rs b/htsget-http-actix/src/lib.rs index 8c9b20c0b..ce9a24a09 100644 --- a/htsget-http-actix/src/lib.rs +++ b/htsget-http-actix/src/lib.rs @@ -62,39 +62,46 @@ pub fn configure_server( /// are supported. pub fn configure_cors(cors: CorsConfig) -> Cors { let mut cors_layer = Cors::default(); - cors_layer = match cors.allow_origins() { - AllowType::Tagged(tagged) => match tagged { - TaggedAllowTypes::Mirror => cors_layer.allow_any_origin(), - TaggedAllowTypes::Any => cors_layer.allow_any_origin().send_wildcard(), - }, - AllowType::List(origins) => { + cors_layer = cors.allow_origins().apply_any( + |cors_layer| cors_layer.allow_any_origin().send_wildcard(), + cors_layer, + ); + cors_layer = cors + .allow_origins() + .apply_mirror(|cors_layer| cors_layer.allow_any_origin(), cors_layer); + cors_layer = cors.allow_origins().apply_list( + |mut cors_layer, origins| { for origin in origins { cors_layer = cors_layer.allowed_origin(&origin.to_string()); } cors_layer - } - }; - - cors_layer = match cors.allow_headers() { - AllowType::Tagged(tagged) => match tagged { - TaggedAllowTypes::Mirror => cors_layer.allow_any_header(), - TaggedAllowTypes::Any => cors_layer.allow_any_header(), - }, - AllowType::List(headers) => cors_layer.allowed_headers(headers.clone()), - }; - - cors_layer = match cors.allow_methods() { - AllowType::Tagged(tagged) => match tagged { - TaggedAllowTypes::Mirror => cors_layer.allow_any_method(), - TaggedAllowTypes::Any => cors_layer.allow_any_method(), }, - AllowType::List(methods) => cors_layer.allowed_methods(methods.clone()), - }; - - cors_layer = match cors.expose_headers() { - AllowType::Tagged(_) => cors_layer.expose_any_header(), - AllowType::List(headers) => cors_layer.expose_headers(headers.clone()), - }; + cors_layer, + ); + + cors_layer = cors + .allow_headers() + .apply_any(|cors_layer| cors_layer.allow_any_header(), cors_layer); + cors_layer = cors.allow_headers().apply_list( + |cors_layer, headers| cors_layer.allowed_headers(headers.clone()), + cors_layer, + ); + + cors_layer = cors + .allow_methods() + .apply_any(|cors_layer| cors_layer.allow_any_method(), cors_layer); + cors_layer = cors.allow_methods().apply_list( + |cors_layer, methods| cors_layer.allowed_methods(methods.clone()), + cors_layer, + ); + + cors_layer = cors + .expose_headers() + .apply_any(|cors_layer| cors_layer.expose_any_header(), cors_layer); + cors_layer = cors.expose_headers().apply_list( + |cors_layer, headers| cors_layer.expose_headers(headers.clone()), + cors_layer, + ); if cors.allow_credentials() { cors_layer = cors_layer.supports_credentials(); diff --git a/htsget-http-core/src/query_builder.rs b/htsget-http-core/src/query_builder.rs index 2742a472b..768446928 100644 --- a/htsget-http-core/src/query_builder.rs +++ b/htsget-http-core/src/query_builder.rs @@ -1,5 +1,5 @@ -use std::collections::HashSet; use htsget_config::{Class, Fields, Format, Tags}; +use std::collections::HashSet; use tracing::instrument; use htsget_config::Query; @@ -343,7 +343,10 @@ mod tests { "part2".to_string() ])) ); - assert_eq!(query.no_tags(), &NoTags(Some(HashSet::from_iter(vec!["part3".to_string()])))); + assert_eq!( + query.no_tags(), + &NoTags(Some(HashSet::from_iter(vec!["part3".to_string()]))) + ); } #[test] @@ -361,6 +364,9 @@ mod tests { "part2".to_string() ])) ); - assert_eq!(query.no_tags(), &NoTags(Some(HashSet::from_iter(vec!["part3".to_string()])))); + assert_eq!( + query.no_tags(), + &NoTags(Some(HashSet::from_iter(vec!["part3".to_string()]))) + ); } } diff --git a/htsget-http-lambda/src/lib.rs b/htsget-http-lambda/src/lib.rs index 7ea630da1..c77edcd94 100644 --- a/htsget-http-lambda/src/lib.rs +++ b/htsget-http-lambda/src/lib.rs @@ -205,7 +205,7 @@ mod tests { use std::sync::Arc; use async_trait::async_trait; - use htsget_config::regex_resolver::{RegexResolver, StorageType, LocalResolver}; + use htsget_config::regex_resolver::{LocalResolver, RegexResolver, StorageType}; use htsget_config::Class; use lambda_http::http::header::HeaderName; use lambda_http::http::Uri; diff --git a/htsget-search/src/htsget/mod.rs b/htsget-search/src/htsget/mod.rs index 0e18a8528..613d8c309 100644 --- a/htsget-search/src/htsget/mod.rs +++ b/htsget-search/src/htsget/mod.rs @@ -231,8 +231,8 @@ impl Response { #[cfg(test)] mod tests { use super::*; - use std::collections::HashSet; use htsget_config::{Fields, NoTags, Tags}; + use std::collections::HashSet; #[test] fn htsget_error_not_found() { @@ -326,11 +326,17 @@ mod tests { #[test] fn query_with_fields() { - let result = Query::new("NA12878", Format::Bam) - .with_fields(Fields::List(HashSet::from_iter(vec!["QNAME".to_string(), "FLAG".to_string()]))); + let result = + Query::new("NA12878", Format::Bam).with_fields(Fields::List(HashSet::from_iter(vec![ + "QNAME".to_string(), + "FLAG".to_string(), + ]))); assert_eq!( result.fields(), - &Fields::List(HashSet::from_iter(vec!["QNAME".to_string(), "FLAG".to_string()])) + &Fields::List(HashSet::from_iter(vec![ + "QNAME".to_string(), + "FLAG".to_string() + ])) ); } @@ -345,7 +351,10 @@ mod tests { let result = Query::new("NA12878", Format::Bam).with_no_tags(vec!["RG", "OQ"]); assert_eq!( result.no_tags(), - &NoTags(Some(HashSet::from_iter(vec!["RG".to_string(), "OQ".to_string()]))) + &NoTags(Some(HashSet::from_iter(vec![ + "RG".to_string(), + "OQ".to_string() + ]))) ); } diff --git a/htsget-search/src/htsget/search.rs b/htsget-search/src/htsget/search.rs index ff2b84752..bf07eeab6 100644 --- a/htsget-search/src/htsget/search.rs +++ b/htsget-search/src/htsget/search.rs @@ -205,7 +205,10 @@ where /// Get the position at the end of file marker. #[instrument(level = "trace", skip(self), ret)] async fn position_at_eof(&self, query: &Query) -> Result { - let file_size = self.get_storage().head(query.format().fmt_file(query.id())).await?; + let file_size = self + .get_storage() + .head(query.format().fmt_file(query.id())) + .await?; Ok( file_size - u64::try_from(self.get_eof_marker().len()) @@ -217,7 +220,10 @@ where #[instrument(level = "trace", skip(self))] async fn read_index(&self, query: &Query) -> Result { trace!("reading index"); - let storage = self.get_storage().get(query.format().fmt_index(query.id()), GetOptions::default()).await?; + let storage = self + .get_storage() + .get(query.format().fmt_index(query.id()), GetOptions::default()) + .await?; Self::read_index_inner(storage) .await .map_err(|err| HtsGetError::io_error(format!("reading {} index: {}", self.get_format(), err))) @@ -289,7 +295,10 @@ where let query_owned = query.clone(); storage_futures.push_back(tokio::spawn(async move { storage - .range_url(query_owned.format().fmt_file(query_owned.id()), RangeUrlOptions::from(range)) + .range_url( + query_owned.format().fmt_file(query_owned.id()), + RangeUrlOptions::from(range), + ) .await })); } @@ -314,7 +323,10 @@ where trace!("getting header"); let get_options = GetOptions::default().with_range(self.get_byte_ranges_for_header(index).await?); - let reader_type = self.get_storage().get(query.format().fmt_file(query.id()), get_options).await?; + let reader_type = self + .get_storage() + .get(query.format().fmt_file(query.id()), get_options) + .await?; let mut reader = Self::init_reader(reader_type); Self::read_raw_header(&mut reader) @@ -408,7 +420,10 @@ where Ok(chunks) }); - let gzi_data = self.get_storage().get(query.format().fmt_gzi(query.id())?, GetOptions::default()).await; + let gzi_data = self + .get_storage() + .get(query.format().fmt_gzi(query.id())?, GetOptions::default()) + .await; let byte_ranges: Vec = match gzi_data { Ok(gzi_data) => { let span = trace_span!("reading gzi"); diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index 7cc948c91..e3d16426b 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -2,7 +2,7 @@ pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketSer #[cfg(feature = "s3-storage")] pub use htsget_config::regex_resolver::aws::S3Resolver; pub use htsget_config::regex_resolver::{ - QueryMatcher, RegexResolver, Resolver, StorageType, LocalResolver, + LocalResolver, QueryMatcher, RegexResolver, Resolver, StorageType, }; pub mod htsget; diff --git a/htsget-search/src/storage/aws.rs b/htsget-search/src/storage/aws.rs index fd2721f32..92974d40f 100644 --- a/htsget-search/src/storage/aws.rs +++ b/htsget-search/src/storage/aws.rs @@ -312,9 +312,7 @@ mod tests { #[tokio::test] async fn existing_key() { with_aws_s3_storage(|storage| async move { - let result = storage - .get("key2", GetOptions::default()) - .await; + let result = storage.get("key2", GetOptions::default()).await; assert!(matches!(result, Ok(_))); }) .await; @@ -323,9 +321,7 @@ mod tests { #[tokio::test] async fn non_existing_key() { with_aws_s3_storage(|storage| async move { - let result = storage - .get("non-existing-key", GetOptions::default()) - .await; + let result = storage.get("non-existing-key", GetOptions::default()).await; assert!(matches!(result, Err(StorageError::AwsS3Error(_, _)))); }) .await; @@ -335,10 +331,7 @@ mod tests { async fn url_of_non_existing_key() { with_aws_s3_storage(|storage| async move { let result = storage - .range_url( - "non-existing-key", - RangeUrlOptions::default(), - ) + .range_url("non-existing-key", RangeUrlOptions::default()) .await; assert!(matches!(result, Err(StorageError::AwsS3Error(_, _)))); }) diff --git a/htsget-search/src/storage/data_server.rs b/htsget-search/src/storage/data_server.rs index b966554af..1db01af00 100644 --- a/htsget-search/src/storage/data_server.rs +++ b/htsget-search/src/storage/data_server.rs @@ -271,7 +271,8 @@ mod tests { test_cors_preflight_request_uri, test_cors_simple_request_uri, }; use htsget_test_utils::http_tests::{ - default_test_config, default_cors_config, Header, Response as TestResponse, TestRequest, TestServer, + default_cors_config, default_test_config, Header, Response as TestResponse, TestRequest, + TestServer, }; use htsget_test_utils::util::generate_test_certificates; diff --git a/htsget-search/src/storage/local.rs b/htsget-search/src/storage/local.rs index 014ea9a90..10276e40b 100644 --- a/htsget-search/src/storage/local.rs +++ b/htsget-search/src/storage/local.rs @@ -36,7 +36,7 @@ impl LocalStorage { .as_ref() .to_path_buf() .canonicalize() - .map_err(|_| StorageError::KeyNotFound(base_path.as_ref().to_string_lossy().to_string())) + .map_err(|_| StorageError::KeyNotFound(base_path.as_ref().to_string_lossy().to_string())) .map(|canonicalized_base_path| Self { base_path: canonicalized_base_path, id_resolver, @@ -162,12 +162,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_forbidden_path() { with_local_storage(|storage| async move { - let result = Storage::get( - &storage, - "folder/../../passwords", - GetOptions::default(), - ) - .await; + let result = Storage::get(&storage, "folder/../../passwords", GetOptions::default()).await; assert!( matches!(result, Err(StorageError::InvalidKey(msg)) if msg == "folder/../../passwords") ); @@ -178,12 +173,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_existing_key() { with_local_storage(|storage| async move { - let result = Storage::get( - &storage, - "folder/../key1", - GetOptions::default(), - ) - .await; + let result = Storage::get(&storage, "folder/../key1", GetOptions::default()).await; assert!(matches!(result, Ok(_))); }) .await; @@ -192,12 +182,8 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_non_existing_key() { with_local_storage(|storage| async move { - let result = Storage::range_url( - &storage, - "non-existing-key", - RangeUrlOptions::default(), - ) - .await; + let result = + Storage::range_url(&storage, "non-existing-key", RangeUrlOptions::default()).await; assert!(matches!(result, Err(StorageError::InvalidKey(msg)) if msg == "non-existing-key")); }) .await; @@ -206,12 +192,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_folder() { with_local_storage(|storage| async move { - let result = Storage::range_url( - &storage, - "folder", - RangeUrlOptions::default(), - ) - .await; + let result = Storage::range_url(&storage, "folder", RangeUrlOptions::default()).await; assert!(matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "folder")); }) .await; @@ -236,12 +217,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_existing_key() { with_local_storage(|storage| async move { - let result = Storage::range_url( - &storage, - "folder/../key1", - RangeUrlOptions::default(), - ) - .await; + let result = Storage::range_url(&storage, "folder/../key1", RangeUrlOptions::default()).await; let expected = Url::new("http://127.0.0.1:8081/data/key1"); assert!(matches!(result, Ok(url) if url == expected)); }) diff --git a/htsget-search/src/storage/mod.rs b/htsget-search/src/storage/mod.rs index 6fe088ad9..90c97e63a 100644 --- a/htsget-search/src/storage/mod.rs +++ b/htsget-search/src/storage/mod.rs @@ -10,7 +10,7 @@ use std::time::Duration; use async_trait::async_trait; use base64::encode; use htsget_config::config::cors::{AllowType, CorsConfig, TaggedAllowTypes, TaggedAnyAllowType}; -use htsget_config::regex_resolver::{Scheme, LocalResolver}; +use htsget_config::regex_resolver::{LocalResolver, Scheme}; use htsget_config::{Class, Query}; use http::{uri, HeaderValue, Method}; use thiserror::Error; @@ -119,39 +119,53 @@ impl UrlFormatter for LocalResolver { /// are supported. pub fn configure_cors(cors: CorsConfig) -> Result { let mut cors_layer = CorsLayer::new(); - cors_layer = match cors.allow_origins() { - AllowType::Tagged(tagged) => match tagged { - TaggedAllowTypes::Mirror => cors_layer.allow_origin(AllowOrigin::mirror_request()), - TaggedAllowTypes::Any => cors_layer.allow_origin(AllowOrigin::any()), - }, - AllowType::List(origins) => cors_layer.allow_origin( - origins - .iter() - .map(|header| header.clone().into_inner()) - .collect::>(), - ), - }; - - cors_layer = match cors.allow_headers() { - AllowType::Tagged(tagged) => match tagged { - TaggedAllowTypes::Mirror => cors_layer.allow_headers(AllowHeaders::mirror_request()), - TaggedAllowTypes::Any => cors_layer.allow_headers(AllowHeaders::any()), - }, - AllowType::List(headers) => cors_layer.allow_headers(headers.clone()), - }; - cors_layer = match cors.allow_methods() { - AllowType::Tagged(tagged) => match tagged { - TaggedAllowTypes::Mirror => cors_layer.allow_methods(AllowMethods::mirror_request()), - TaggedAllowTypes::Any => cors_layer.allow_methods(AllowMethods::any()), + cors_layer = cors.allow_origins().apply_any( + |cors_layer| cors_layer.allow_origin(AllowOrigin::any()), + cors_layer, + ); + cors_layer = cors.allow_origins().apply_mirror( + |cors_layer| cors_layer.allow_origin(AllowOrigin::mirror_request()), + cors_layer, + ); + cors_layer = cors.allow_origins().apply_list( + |cors_layer, origins| { + cors_layer.allow_origin( + origins + .iter() + .map(|header| header.clone().into_inner()) + .collect::>(), + ) }, - AllowType::List(methods) => cors_layer.allow_methods(methods.clone()), - }; - - cors_layer = match cors.expose_headers() { - AllowType::Tagged(_) => cors_layer, - AllowType::List(headers) => cors_layer.expose_headers(headers.clone()), - }; + cors_layer, + ); + + cors_layer = cors.allow_headers().apply_any( + |cors_layer| cors_layer.allow_headers(AllowHeaders::mirror_request()), + cors_layer, + ); + cors_layer = cors.allow_headers().apply_list( + |cors_layer, headers| cors_layer.allow_headers(headers.clone()), + cors_layer, + ); + + cors_layer = cors.allow_methods().apply_any( + |cors_layer| cors_layer.allow_methods(AllowMethods::mirror_request()), + cors_layer, + ); + cors_layer = cors.allow_methods().apply_list( + |cors_layer, methods| cors_layer.allow_methods(methods.clone()), + cors_layer, + ); + + cors_layer = cors.expose_headers().apply_any( + |cors_layer| cors_layer.expose_headers(ExposeHeaders::any()), + cors_layer, + ); + cors_layer = cors.expose_headers().apply_list( + |cors_layer, headers| cors_layer.expose_headers(headers.clone()), + cors_layer, + ); Ok( cors_layer diff --git a/htsget-test-utils/src/cors_tests.rs b/htsget-test-utils/src/cors_tests.rs index 78a5df0e8..d444841b5 100644 --- a/htsget-test-utils/src/cors_tests.rs +++ b/htsget-test-utils/src/cors_tests.rs @@ -70,7 +70,8 @@ pub async fn test_cors_preflight_request_uri( .get(ACCESS_CONTROL_ALLOW_ORIGIN) .unwrap() .to_str() - .unwrap().to_lowercase(), + .unwrap() + .to_lowercase(), "http://example.com" ); @@ -81,16 +82,16 @@ pub async fn test_cors_preflight_request_uri( .unwrap() .to_str() .unwrap() - .to_lowercase(), + .to_lowercase(), "x-requested-with" ); - assert!( - response - .headers - .get(ACCESS_CONTROL_ALLOW_METHODS) - .unwrap() - .to_str() - .unwrap().to_lowercase().contains("post") - ); + assert!(response + .headers + .get(ACCESS_CONTROL_ALLOW_METHODS) + .unwrap() + .to_str() + .unwrap() + .to_lowercase() + .contains("post")); } diff --git a/htsget-test-utils/src/http_tests.rs b/htsget-test-utils/src/http_tests.rs index d89322dd0..7c1801c58 100644 --- a/htsget-test-utils/src/http_tests.rs +++ b/htsget-test-utils/src/http_tests.rs @@ -7,8 +7,8 @@ use async_trait::async_trait; use htsget_config::config::cors::{AllowType, CorsConfig}; use htsget_config::config::{DataServerConfig, TicketServerConfig}; use htsget_config::regex_resolver::{LocalResolver, RegexResolver, Scheme, StorageType}; -use http::HeaderMap; use http::uri::Authority; +use http::HeaderMap; use serde::de; use crate::util::generate_test_certificates; @@ -105,12 +105,7 @@ pub fn default_test_resolver(addr: SocketAddr, scheme: Scheme) -> RegexResolver resolver.set_authority(Authority::from_str(&addr.to_string()).unwrap()); resolver.set_scheme(scheme); - RegexResolver::new( - StorageType::Local(resolver), - ".*", - "$0", - Default::default() - ).unwrap() + RegexResolver::new(StorageType::Local(resolver), ".*", "$0", Default::default()).unwrap() } /// Default config with fixed port. diff --git a/htsget-test-utils/src/server_tests.rs b/htsget-test-utils/src/server_tests.rs index a08de80af..073987d71 100644 --- a/htsget-test-utils/src/server_tests.rs +++ b/htsget-test-utils/src/server_tests.rs @@ -53,7 +53,8 @@ pub async fn test_response(response: Response, class: Class) { .unwrap(), ) .send() - .await.unwrap() + .await + .unwrap() .bytes() .await .unwrap() @@ -79,7 +80,11 @@ pub async fn test_response(response: Response, class: Class) { /// Create the a [HttpTicketFormatter], spawn the ticket server, returning the expected path and the formatter. pub async fn formatter_and_expected_path(config: &Config) -> (String, HttpTicketFormatter) { let mut formatter = formatter_from_config(config).unwrap(); - spawn_ticket_server(config.data_server().unwrap().local_path().into(), &mut formatter).await; + spawn_ticket_server( + config.data_server().unwrap().local_path().into(), + &mut formatter, + ) + .await; (expected_url_path(&formatter), formatter) } From 77a6caa38639d366b4f9c0fda279fa0f4bd7be62 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 21 Dec 2022 09:41:53 +1100 Subject: [PATCH 30/45] config: remove setters, add constructors, add documentation. --- htsget-config/src/config/cors.rs | 45 +++----- htsget-config/src/config/mod.rs | 131 +++++++++++++----------- htsget-config/src/lib.rs | 1 + htsget-config/src/regex_resolver/aws.rs | 3 +- htsget-config/src/regex_resolver/mod.rs | 62 +++++++---- htsget-test-utils/src/http_tests.rs | 100 ++++++++---------- 6 files changed, 173 insertions(+), 169 deletions(-) diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs index fe70706fc..d02b26cb2 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/cors.rs @@ -10,7 +10,7 @@ use std::str::FromStr; /// The maximum default amount of time a CORS request can be cached for in seconds. const CORS_MAX_AGE: usize = 86400; -/// Tagged allow headers for cors config. Either Mirror or Any. +/// Tagged allow headers for cors config, either Mirror or Any. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub enum TaggedAllowTypes { #[serde(alias = "mirror", alias = "MIRROR")] @@ -19,15 +19,14 @@ pub enum TaggedAllowTypes { Any, } -/// Tagged allow headers for cors config. Either Mirror or Any. +/// Tagged Any allow type for cors config. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub enum TaggedAnyAllowType { #[serde(alias = "any", alias = "ANY")] Any, } -/// Allowed header for cors config. Any allows all headers by sending a wildcard, -/// and mirror allows all headers by mirroring the received headers. +/// Allowed type for cors config which is used to configure cors behaviour. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(untagged)] pub enum AllowType { @@ -122,6 +121,7 @@ where .collect() } +/// A wrapper around a http HeaderValue which is used to implement FromStr and Display. #[derive(Debug, Clone, PartialEq, Eq)] pub struct HeaderValue(HeaderValueInner); @@ -145,7 +145,7 @@ impl Display for HeaderValue { } } -/// Configuration for the htsget server. +/// Cors configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct CorsConfig { @@ -158,53 +158,40 @@ pub struct CorsConfig { } impl CorsConfig { + /// Create new cors config. + pub fn new(allow_credentials: bool, allow_origins: AllowType, allow_headers: AllowType, allow_methods: AllowType, max_age: usize, expose_headers: AllowType) -> Self { + Self { allow_credentials, allow_origins, allow_headers, allow_methods, max_age, expose_headers } + } + + /// Get allow credentials. pub fn allow_credentials(&self) -> bool { self.allow_credentials } + /// Get allow origins. pub fn allow_origins(&self) -> &AllowType { &self.allow_origins } + /// Get allow headers. pub fn allow_headers(&self) -> &AllowType { &self.allow_headers } + /// Get allow methods. pub fn allow_methods(&self) -> &AllowType { &self.allow_methods } + /// Get max age. pub fn max_age(&self) -> usize { self.max_age } + /// Get expose headers. pub fn expose_headers(&self) -> &AllowType { &self.expose_headers } - - pub fn set_allow_credentials(&mut self, allow_credentials: bool) { - self.allow_credentials = allow_credentials; - } - - pub fn set_allow_origins(&mut self, allow_origins: AllowType) { - self.allow_origins = allow_origins; - } - - pub fn set_allow_headers(&mut self, allow_headers: AllowType) { - self.allow_headers = allow_headers; - } - - pub fn set_allow_methods(&mut self, allow_methods: AllowType) { - self.allow_methods = allow_methods; - } - - pub fn set_max_age(&mut self, max_age: usize) { - self.max_age = max_age; - } - - pub fn set_expose_headers(&mut self, expose_headers: AllowType) { - self.expose_headers = expose_headers; - } } impl Default for CorsConfig { diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 39ec79c1f..ae05c3026 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -91,7 +91,7 @@ struct Args { config: Option, } -/// Configuration for the server. Each field will be read from environment variables. +/// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct Config { @@ -101,14 +101,12 @@ pub struct Config { resolvers: Vec, } -/// None component of data server config. Allows deserializing no data server config as none. #[derive(Serialize, Deserialize, Debug, Clone)] enum DataServerConfigNone { #[serde(alias = "none", alias = "NONE", alias = "")] None, } -/// Data server config enum options. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] enum DataServerConfigOption { @@ -118,7 +116,7 @@ enum DataServerConfigOption { with_prefix!(ticket_server_prefix "ticket_server_"); -/// Configuration for the htsget server. +/// Configuration for the htsget ticket server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { @@ -130,93 +128,105 @@ pub struct TicketServerConfig { } impl TicketServerConfig { + /// Create a new ticket server config. + pub fn new(ticket_server_addr: SocketAddr, cors: CorsConfig, service_info: ServiceInfo) -> Self { + Self { ticket_server_addr, cors, service_info } + } + + /// Get the addr. pub fn addr(&self) -> SocketAddr { self.ticket_server_addr } + /// Get cors config. pub fn cors(&self) -> &CorsConfig { &self.cors } + /// Get service info. pub fn service_info(&self) -> &ServiceInfo { &self.service_info } + /// Get allow credentials. pub fn allow_credentials(&self) -> bool { self.cors.allow_credentials() } + /// Get allow origins. pub fn allow_origins(&self) -> &AllowType { self.cors.allow_origins() } + /// Get allow headers. pub fn allow_headers(&self) -> &AllowType { self.cors.allow_headers() } + /// Get allow methods. pub fn allow_methods(&self) -> &AllowType { self.cors.allow_methods() } + /// Get max age. pub fn max_age(&self) -> usize { self.cors.max_age() } + /// Get expose headers. pub fn expose_headers(&self) -> &AllowType { self.cors.expose_headers() } + /// Get id. pub fn id(&self) -> Option<&str> { self.service_info.id() } + /// Get name. pub fn name(&self) -> Option<&str> { self.service_info.name() } + /// Get version. pub fn version(&self) -> Option<&str> { self.service_info.version() } + /// Get organization name. pub fn organization_name(&self) -> Option<&str> { self.service_info.organization_name() } + /// Get the organization url. pub fn organization_url(&self) -> Option<&str> { self.service_info.organization_url() } + /// Get the contact url. pub fn contact_url(&self) -> Option<&str> { self.service_info.contact_url() } + /// Get the documentation url. pub fn documentation_url(&self) -> Option<&str> { self.service_info.documentation_url() } + /// Get created at. pub fn created_at(&self) -> Option<&str> { self.service_info.created_at() } + /// Get updated at. pub fn updated_at(&self) -> Option<&str> { self.service_info.updated_at() } + /// Get the environment. pub fn environment(&self) -> Option<&str> { self.service_info.environment() } - - pub fn set_ticket_server_addr(&mut self, ticket_server_addr: SocketAddr) { - self.ticket_server_addr = ticket_server_addr; - } - - pub fn set_cors(&mut self, cors: CorsConfig) { - self.cors = cors; - } - - pub fn set_service_info(&mut self, service_info: ServiceInfo) { - self.service_info = service_info; - } } /// Configuration for the htsget server. @@ -233,77 +243,70 @@ pub struct DataServerConfig { } impl DataServerConfig { + /// Create a new data server config. + pub fn new(addr: SocketAddr, local_path: PathBuf, serve_at: PathBuf, key: Option, cert: Option, cors: CorsConfig) -> Self { + Self { addr, local_path, serve_at, key, cert, cors } + } + + /// Get the address. pub fn addr(&self) -> SocketAddr { self.addr } + /// Get the local path. pub fn local_path(&self) -> &Path { &self.local_path } + /// Get the serve at path. pub fn serve_at(&self) -> &Path { &self.serve_at } + /// Get the key. pub fn key(&self) -> Option<&Path> { self.key.as_deref() } + /// Get the cert. pub fn cert(&self) -> Option<&Path> { self.cert.as_deref() } + /// Get cors config. pub fn cors(&self) -> &CorsConfig { &self.cors } + /// Get allow credentials. pub fn allow_credentials(&self) -> bool { self.cors.allow_credentials() } + /// Get allow origins. pub fn allow_origins(&self) -> &AllowType { self.cors.allow_origins() } + /// Get allow headers. pub fn allow_headers(&self) -> &AllowType { self.cors.allow_headers() } + /// Get allow methods. pub fn allow_methods(&self) -> &AllowType { self.cors.allow_methods() } + /// Get the max age. pub fn max_age(&self) -> usize { self.cors.max_age() } + /// Get the expose headers. pub fn expose_headers(&self) -> &AllowType { self.cors.expose_headers() } - - pub fn set_addr(&mut self, addr: SocketAddr) { - self.addr = addr; - } - - pub fn set_local_path(&mut self, path: PathBuf) { - self.local_path = path; - } - - pub fn set_serve_at(&mut self, serve_at: PathBuf) { - self.serve_at = serve_at; - } - - pub fn set_key(&mut self, key: Option) { - self.key = key; - } - - pub fn set_cert(&mut self, cert: Option) { - self.cert = cert; - } - - pub fn set_cors(&mut self, cors: CorsConfig) { - self.cors = cors; - } } impl Default for DataServerConfig { @@ -338,42 +341,52 @@ pub struct ServiceInfo { } impl ServiceInfo { + /// Get the id. pub fn id(&self) -> Option<&str> { self.id.as_deref() } + /// Get the name. pub fn name(&self) -> Option<&str> { self.name.as_deref() } + /// Get the version. pub fn version(&self) -> Option<&str> { self.version.as_deref() } + /// Get the organization name. pub fn organization_name(&self) -> Option<&str> { self.organization_name.as_deref() } + /// Get the organization url. pub fn organization_url(&self) -> Option<&str> { self.organization_url.as_deref() } + /// Get the contact url. pub fn contact_url(&self) -> Option<&str> { self.contact_url.as_deref() } + /// Get the documentation url. pub fn documentation_url(&self) -> Option<&str> { self.documentation_url.as_deref() } + /// Get created at. pub fn created_at(&self) -> Option<&str> { self.created_at.as_deref() } + /// Get updated at. pub fn updated_at(&self) -> Option<&str> { self.updated_at.as_deref() } + /// Get environment. pub fn environment(&self) -> Option<&str> { self.environment.as_deref() } @@ -400,6 +413,21 @@ impl Default for Config { } impl Config { + /// Create a new config. + pub fn new(ticket_server: TicketServerConfig, data_server: Option, resolvers: Vec) -> Self { + Self { + ticket_server, + data_server: match data_server { + None => { + DataServerConfigOption::None(DataServerConfigNone::None) + } + Some(value) => { + DataServerConfigOption::Some(value) + } + }, + resolvers } + } + /// Parse the command line arguments pub fn parse_args() -> PathBuf { Args::parse().config.unwrap_or_else(|| "".into()) @@ -437,10 +465,12 @@ impl Config { Ok(()) } + /// Get the ticket server. pub fn ticket_server(&self) -> &TicketServerConfig { &self.ticket_server } + /// Get the data server. pub fn data_server(&self) -> Option<&DataServerConfig> { match self.data_server { DataServerConfigOption::None(_) => None, @@ -448,32 +478,15 @@ impl Config { } } + /// Get the resolvers. pub fn resolvers(&self) -> &[RegexResolver] { &self.resolvers } + /// Get owned resolvers. pub fn owned_resolvers(self) -> Vec { self.resolvers } - - pub fn set_ticket_server(&mut self, ticket_server: TicketServerConfig) { - self.ticket_server = ticket_server; - } - - pub fn set_data_server(&mut self, data_server: Option) { - match data_server { - None => { - self.data_server = DataServerConfigOption::None(DataServerConfigNone::None); - } - Some(value) => { - self.data_server = DataServerConfigOption::Some(value); - } - } - } - - pub fn set_resolvers(&mut self, resolvers: Vec) { - self.resolvers = resolvers; - } } #[cfg(test)] diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 385e38dae..44adba8d4 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -76,6 +76,7 @@ impl fmt::Display for Format { } } +/// Class component of htsget response. #[derive(Copy, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] #[serde(rename_all(serialize = "lowercase"))] pub enum Class { diff --git a/htsget-config/src/regex_resolver/aws.rs b/htsget-config/src/regex_resolver/aws.rs index bb25fd826..b81ce6e7b 100644 --- a/htsget-config/src/regex_resolver/aws.rs +++ b/htsget-config/src/regex_resolver/aws.rs @@ -1,7 +1,7 @@ use serde; use serde::{Deserialize, Serialize}; -/// Configuration for the htsget server. +/// S3 configuration for the htsget server. #[derive(Deserialize, Serialize, Debug, Clone, Default)] #[serde(default)] pub struct S3Resolver { @@ -9,6 +9,7 @@ pub struct S3Resolver { } impl S3Resolver { + /// Get the bucket. pub fn bucket(&self) -> &str { &self.bucket } diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index f4036aeff..5144dd470 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -42,6 +42,7 @@ impl Default for StorageType { } } +/// Schemes that can be used with htsget. #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] pub enum Scheme { #[serde(alias = "http", alias = "HTTP")] @@ -56,7 +57,7 @@ impl Default for Scheme { } } -/// Configuration for the htsget server. +/// A local resolver, which can return files from the local file system. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct LocalResolver { @@ -68,37 +69,40 @@ pub struct LocalResolver { } impl LocalResolver { + /// Create a local resolver. + pub fn new( + scheme: Scheme, + authority: Authority, + local_path: String, + path_prefix: String + ) -> Self { + Self { + scheme, + authority, + local_path, + path_prefix + } + } + + /// Get the scheme. pub fn scheme(&self) -> Scheme { self.scheme } + /// Get the authority. pub fn authority(&self) -> &Authority { &self.authority } + /// Get the local path. pub fn local_path(&self) -> &str { &self.local_path } + /// Get the path prefix. pub fn path_prefix(&self) -> &str { &self.path_prefix } - - pub fn set_scheme(&mut self, scheme: Scheme) { - self.scheme = scheme; - } - - pub fn set_authority(&mut self, authority: Authority) { - self.authority = authority; - } - - pub fn set_local_path(&mut self, local_path: String) { - self.local_path = local_path; - } - - pub fn set_path_prefix(&mut self, path_prefix: String) { - self.path_prefix = path_prefix; - } } impl Default for LocalResolver { @@ -124,7 +128,7 @@ pub struct RegexResolver { storage_type: StorageType, } -/// A query that can be matched with the regex resolver. +/// A query guard represents query parameters that can be allowed to resolver for a given query. #[derive(Serialize, Clone, Debug, Deserialize)] #[serde(default)] pub struct QueryGuard { @@ -136,7 +140,7 @@ pub struct QueryGuard { allow_tags: Tags, } -/// Referneces names that can be matched. +/// Reference names that can be matched. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(untagged)] pub enum ReferenceNames { @@ -146,26 +150,32 @@ pub enum ReferenceNames { } impl QueryGuard { + /// Get allow formats. pub fn allow_formats(&self) -> &[Format] { &self.allow_formats } + /// Get allow classes. pub fn allow_classes(&self) -> &[Class] { &self.allow_classes } + /// Get allow reference names. pub fn allow_reference_names(&self) -> &ReferenceNames { &self.allow_reference_names } + /// Get allow interval. pub fn allow_interval(&self) -> Interval { self.allow_interval } + /// Get allow fields. pub fn allow_fields(&self) -> &Fields { &self.allow_fields } + /// Get allow tags. pub fn allow_tags(&self) -> &Tags { &self.allow_tags } @@ -255,42 +265,52 @@ impl RegexResolver { }) } + /// Get the regex. pub fn regex(&self) -> &Regex { &self.regex } - + + /// Get the substitution string. pub fn substitution_string(&self) -> &str { &self.substitution_string } + /// Get the query guard. pub fn guard(&self) -> &QueryGuard { &self.guard } + /// Get the storage type. pub fn storage_type(&self) -> &StorageType { &self.storage_type } + /// Get allow formats. pub fn allow_formats(&self) -> &[Format] { self.guard.allow_formats() } + /// Get allow classes. pub fn allow_classes(&self) -> &[Class] { self.guard.allow_classes() } + /// Get allow reference names. pub fn allow_reference_names(&self) -> &ReferenceNames { &self.guard.allow_reference_names } + /// Get allow interval. pub fn allow_interval(&self) -> Interval { self.guard.allow_interval } + /// Get allow fields. pub fn allow_fields(&self) -> &Fields { &self.guard.allow_fields } + /// Get allow tags. pub fn allow_tags(&self) -> &Tags { &self.guard.allow_tags } @@ -318,7 +338,7 @@ pub mod tests { #[test] fn resolver_resolve_id() { - let mut resolver = RegexResolver::new( + let resolver = RegexResolver::new( StorageType::default(), ".*", "$0-test", diff --git a/htsget-test-utils/src/http_tests.rs b/htsget-test-utils/src/http_tests.rs index 7c1801c58..4b352e732 100644 --- a/htsget-test-utils/src/http_tests.rs +++ b/htsget-test-utils/src/http_tests.rs @@ -4,7 +4,7 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use async_trait::async_trait; -use htsget_config::config::cors::{AllowType, CorsConfig}; +use htsget_config::config::cors::{AllowType, CorsConfig, TaggedAnyAllowType}; use htsget_config::config::{DataServerConfig, TicketServerConfig}; use htsget_config::regex_resolver::{LocalResolver, RegexResolver, Scheme, StorageType}; use http::uri::Authority; @@ -89,38 +89,23 @@ pub fn default_dir_data() -> PathBuf { default_dir().join("data") } -fn set_path(config: &mut DataServerConfig) { - config.set_local_path(default_dir_data()); -} - -fn set_addr_and_path(config: &mut DataServerConfig, addr: SocketAddr) { - set_path(config); - config.set_addr(addr); -} - /// Get the default test resolver. pub fn default_test_resolver(addr: SocketAddr, scheme: Scheme) -> RegexResolver { - let mut resolver = LocalResolver::default(); - resolver.set_local_path(default_dir_data().to_str().unwrap().to_string()); - resolver.set_authority(Authority::from_str(&addr.to_string()).unwrap()); - resolver.set_scheme(scheme); + let resolver = LocalResolver::new( + scheme, + Authority::from_str(&addr.to_string()).unwrap(), + default_dir_data().to_str().unwrap().to_string(), + "/data".to_string() + ); RegexResolver::new(StorageType::Local(resolver), ".*", "$0", Default::default()).unwrap() } /// Default config with fixed port. pub fn default_config_fixed_port() -> Config { - let mut config = Config::default(); - - let mut data_server_config = DataServerConfig::default(); - let addr = data_server_config.addr(); - set_path(&mut data_server_config); + let addr = "127.0.0.1:8081".parse().unwrap(); - config.set_data_server(Some(data_server_config)); - - config.set_resolvers(vec![default_test_resolver(addr, Scheme::Http)]); - - config + default_test_config_params(addr, None, None, Scheme::Http) } fn get_dynamic_addr() -> SocketAddr { @@ -130,54 +115,51 @@ fn get_dynamic_addr() -> SocketAddr { /// Set the default cors testing config. pub fn default_cors_config() -> CorsConfig { - let mut cors = CorsConfig::default(); - - cors.set_allow_credentials(false); - cors.set_allow_origins(AllowType::List(vec!["http://example.com".parse().unwrap()])); - - cors + CorsConfig::new( + false, + AllowType::List(vec!["http://example.com".parse().unwrap()]), + AllowType::Tagged(TaggedAnyAllowType::Any), + AllowType::Tagged(TaggedAnyAllowType::Any), + 1000, + AllowType::List(vec![]), + ) +} + +fn default_test_config_params(addr: SocketAddr, key: Option, cert: Option, scheme: Scheme) -> Config { + let cors = default_cors_config(); + let server_config = DataServerConfig::new( + addr, + default_dir_data(), + PathBuf::from("/data"), + key, + cert, + cors.clone() + ); + + Config::new( + TicketServerConfig::new( + "127.0.0.1:8080".parse().unwrap(), + cors, + Default::default() + ), + Some(server_config), + vec![default_test_resolver(addr, scheme)] + ) } /// Default config using the current cargo manifest directory, and dynamic port. pub fn default_test_config() -> Config { - let mut server_config = DataServerConfig::default(); let addr = get_dynamic_addr(); - set_addr_and_path(&mut server_config, addr); - - let mut cors = default_cors_config(); - server_config.set_cors(cors.clone()); - - let mut config = Config::default(); - let mut ticket_server_config = TicketServerConfig::default(); - ticket_server_config.set_cors(cors); - - config.set_ticket_server(ticket_server_config); - config.set_data_server(Some(server_config)); - config.set_resolvers(vec![default_test_resolver(addr, Scheme::Http)]); - - config + default_test_config_params(addr, None, None, Scheme::Http) } /// Config with tls ticket server, using the current cargo manifest directory. pub fn config_with_tls>(path: P) -> Config { - let mut server_config = DataServerConfig::default(); let addr = get_dynamic_addr(); - - set_addr_and_path(&mut server_config, addr); - let (key_path, cert_path) = generate_test_certificates(path, "key.pem", "cert.pem"); - server_config.set_key(Some(key_path)); - server_config.set_cert(Some(cert_path)); - - let mut config = Config::default(); - - config.set_data_server(Some(server_config)); - - config.set_resolvers(vec![default_test_resolver(addr, Scheme::Https)]); - - config + default_test_config_params(addr, Some(key_path), Some(cert_path), Scheme::Https) } /// Get the event associated with the file. From 4819bd3d464fc947e0327727b6f49f4f87a31fd1 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 21 Dec 2022 13:55:39 +1100 Subject: [PATCH 31/45] config: allow specifying tags, reference names, fields with an 'All' value --- htsget-config/config.toml | 69 +++++++++++------ htsget-config/src/config/cors.rs | 68 +++++++++-------- htsget-config/src/config/mod.rs | 81 ++++++++++++-------- htsget-config/src/lib.rs | 17 +++-- htsget-config/src/regex_resolver/mod.rs | 99 +++++++++++++------------ htsget-search/src/htsget/mod.rs | 6 +- htsget-search/src/storage/mod.rs | 2 +- htsget-test-utils/src/http_tests.rs | 26 ++++--- 8 files changed, 216 insertions(+), 152 deletions(-) diff --git a/htsget-config/config.toml b/htsget-config/config.toml index e74a53fdf..cd96af9e3 100644 --- a/htsget-config/config.toml +++ b/htsget-config/config.toml @@ -1,28 +1,51 @@ -ticket_server_addr = "127.0.0.1:8082" +#ticket_server_addr = '127.0.0.1:8080' #ticket_server_cors_allow_credentials = false -#ticket_server_cors_allow_origin = "http://localhost:8080" -#start_data_server = true -#data_server_path = "data" -#data_server_serve_at = "/data" -#data_server_config = "None" -#data_server_config = [] -#data_server_addr = "127.0.0.1:8082" -#data_server_cors_allow_credentials = false -#data_server_cors_allow_origins = ["http://localhost:8081"] -#data_server_cors_allow_methods = "Any" +#ticket_server_cors_allow_origins = ['http://localhost:8080'] +#ticket_server_cors_allow_headers = 'Any' +#ticket_server_cors_allow_methods = 'Any' +#ticket_server_cors_max_age = 86400 +#ticket_server_cors_expose_headers = [] # -#[[resolver]] -#regex = ".*" -#substitution_string = "$0" +## To disable the data server: +## data_server = '' +#[data_server] +#addr = '127.0.0.1:8081' +#local_path = 'data' +#serve_at = '/data' +#cors_allow_credentials = false +#cors_allow_origins = ['http://localhost:8080'] +#cors_allow_headers = 'Any' +#cors_allow_methods = 'Any' +#cors_max_age = 86400 +#cors_expose_headers = [] # -#storage_type.type = "Url" -#storage_type.scheme = "Https" -#storage_type.authority = "127.0.0.1:8081" -#storage_type.path = "/data" +#[[resolvers]] +#regex = '.*' +#substitution_string = '$0' # -#[resolver.guard] -#match_formats = ["BAM"] -#start_interval.start = 100 -#match_fields = ["field1"] -#match_no_tags = ["tag1"] +#[resolvers.guard] +#allow_formats = [ +# 'BAM', +# 'CRAM', +# 'VCF', +# 'BCF', +#] +#allow_classes = [ +# 'body', +# 'header', +#] # +##allow_interval.start = 0 +##allow_interval.end = 100 +# +## Default is to allow all reference names, fields, and tags. +##allow_reference_names = ['chr1'] +##allow_fields = ['QNAME'] +##allow_tags = ['RG'] +# +#[resolvers.storage_type] +#type = 'Local' +#scheme = 'Http' +#authority = '127.0.0.1:8081' +#local_path = 'data' +#path_prefix = '/data' diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs index d02b26cb2..0f88cbf1d 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/cors.rs @@ -1,4 +1,5 @@ use crate::config::default_server_origin; +use crate::TaggedTypeAll; use http::header::{HeaderName, HeaderValue as HeaderValueInner, InvalidHeaderValue}; use http::Method; use serde::de::Error; @@ -15,21 +16,14 @@ const CORS_MAX_AGE: usize = 86400; pub enum TaggedAllowTypes { #[serde(alias = "mirror", alias = "MIRROR")] Mirror, - #[serde(alias = "any", alias = "ANY")] - Any, -} - -/// Tagged Any allow type for cors config. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub enum TaggedAnyAllowType { - #[serde(alias = "any", alias = "ANY")] - Any, + #[serde(alias = "all", alias = "ALL")] + All, } /// Allowed type for cors config which is used to configure cors behaviour. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(untagged)] -pub enum AllowType { +pub enum AllowType { Tagged(Tagged), #[serde(bound(serialize = "T: Display", deserialize = "T: FromStr, T::Err: Display"))] #[serde( @@ -82,17 +76,17 @@ impl AllowType { where F: FnOnce(U) -> U, { - self.apply_tagged(func, builder, &TaggedAllowTypes::Any) + self.apply_tagged(func, builder, &TaggedAllowTypes::All) } } -impl AllowType { +impl AllowType { /// Apply a function to the builder when the type is Any. pub fn apply_any(&self, func: F, builder: U) -> U where F: FnOnce(U) -> U, { - self.apply_tagged(func, builder, &TaggedAnyAllowType::Any) + self.apply_tagged(func, builder, &TaggedTypeAll::All) } } @@ -150,36 +144,50 @@ impl Display for HeaderValue { #[serde(default)] pub struct CorsConfig { allow_credentials: bool, - allow_origins: AllowType, - allow_headers: AllowType, - allow_methods: AllowType, + allow_origins: AllowType, + allow_headers: AllowType, + allow_methods: AllowType, max_age: usize, - expose_headers: AllowType, + expose_headers: AllowType, } impl CorsConfig { /// Create new cors config. - pub fn new(allow_credentials: bool, allow_origins: AllowType, allow_headers: AllowType, allow_methods: AllowType, max_age: usize, expose_headers: AllowType) -> Self { - Self { allow_credentials, allow_origins, allow_headers, allow_methods, max_age, expose_headers } + pub fn new( + allow_credentials: bool, + allow_origins: AllowType, + allow_headers: AllowType, + allow_methods: AllowType, + max_age: usize, + expose_headers: AllowType, + ) -> Self { + Self { + allow_credentials, + allow_origins, + allow_headers, + allow_methods, + max_age, + expose_headers, + } } - + /// Get allow credentials. pub fn allow_credentials(&self) -> bool { self.allow_credentials } /// Get allow origins. - pub fn allow_origins(&self) -> &AllowType { + pub fn allow_origins(&self) -> &AllowType { &self.allow_origins } /// Get allow headers. - pub fn allow_headers(&self) -> &AllowType { + pub fn allow_headers(&self) -> &AllowType { &self.allow_headers } /// Get allow methods. - pub fn allow_methods(&self) -> &AllowType { + pub fn allow_methods(&self) -> &AllowType { &self.allow_methods } @@ -189,7 +197,7 @@ impl CorsConfig { } /// Get expose headers. - pub fn expose_headers(&self) -> &AllowType { + pub fn expose_headers(&self) -> &AllowType { &self.expose_headers } } @@ -201,8 +209,8 @@ impl Default for CorsConfig { allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static( default_server_origin(), ))]), - allow_headers: AllowType::Tagged(TaggedAnyAllowType::Any), - allow_methods: AllowType::Tagged(TaggedAnyAllowType::Any), + allow_headers: AllowType::Tagged(TaggedTypeAll::All), + allow_methods: AllowType::Tagged(TaggedTypeAll::All), max_age: CORS_MAX_AGE, expose_headers: AllowType::List(vec![]), } @@ -232,8 +240,8 @@ mod tests { #[test] fn unit_variant_any_allow_type() { test_cors_config( - "allow_methods = \"Any\"", - &AllowType::Tagged(TaggedAnyAllowType::Any), + "allow_methods = \"All\"", + &AllowType::Tagged(TaggedTypeAll::All), |config| config.allow_methods(), ); } @@ -259,8 +267,8 @@ mod tests { #[test] fn tagged_any_allow_type() { test_cors_config( - "expose_headers = \"Any\"", - &AllowType::Tagged(TaggedAnyAllowType::Any), + "expose_headers = \"All\"", + &AllowType::Tagged(TaggedTypeAll::All), |config| config.expose_headers(), ); } diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index ae05c3026..ebe5c8035 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -7,7 +7,7 @@ use std::iter::Map; use std::net::SocketAddr; use std::path::{Path, PathBuf}; -use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAnyAllowType}; +use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAllowTypes}; use clap::Parser; use figment::providers::{Env, Format, Serialized, Toml}; use figment::value::Value::Dict; @@ -114,14 +114,14 @@ enum DataServerConfigOption { Some(DataServerConfig), } -with_prefix!(ticket_server_prefix "ticket_server_"); +with_prefix!(ticket_server_cors_prefix "ticket_server_cors_"); /// Configuration for the htsget ticket server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { ticket_server_addr: SocketAddr, - #[serde(flatten, with = "ticket_server_prefix")] + #[serde(flatten, with = "ticket_server_cors_prefix")] cors: CorsConfig, #[serde(flatten)] service_info: ServiceInfo, @@ -130,7 +130,11 @@ pub struct TicketServerConfig { impl TicketServerConfig { /// Create a new ticket server config. pub fn new(ticket_server_addr: SocketAddr, cors: CorsConfig, service_info: ServiceInfo) -> Self { - Self { ticket_server_addr, cors, service_info } + Self { + ticket_server_addr, + cors, + service_info, + } } /// Get the addr. @@ -154,17 +158,17 @@ impl TicketServerConfig { } /// Get allow origins. - pub fn allow_origins(&self) -> &AllowType { + pub fn allow_origins(&self) -> &AllowType { self.cors.allow_origins() } /// Get allow headers. - pub fn allow_headers(&self) -> &AllowType { + pub fn allow_headers(&self) -> &AllowType { self.cors.allow_headers() } /// Get allow methods. - pub fn allow_methods(&self) -> &AllowType { + pub fn allow_methods(&self) -> &AllowType { self.cors.allow_methods() } @@ -174,7 +178,7 @@ impl TicketServerConfig { } /// Get expose headers. - pub fn expose_headers(&self) -> &AllowType { + pub fn expose_headers(&self) -> &AllowType { self.cors.expose_headers() } @@ -229,6 +233,8 @@ impl TicketServerConfig { } } +with_prefix!(cors_prefix "cors_"); + /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] @@ -238,14 +244,28 @@ pub struct DataServerConfig { serve_at: PathBuf, key: Option, cert: Option, - #[serde(flatten)] + #[serde(flatten, with = "cors_prefix")] cors: CorsConfig, } impl DataServerConfig { /// Create a new data server config. - pub fn new(addr: SocketAddr, local_path: PathBuf, serve_at: PathBuf, key: Option, cert: Option, cors: CorsConfig) -> Self { - Self { addr, local_path, serve_at, key, cert, cors } + pub fn new( + addr: SocketAddr, + local_path: PathBuf, + serve_at: PathBuf, + key: Option, + cert: Option, + cors: CorsConfig, + ) -> Self { + Self { + addr, + local_path, + serve_at, + key, + cert, + cors, + } } /// Get the address. @@ -284,17 +304,17 @@ impl DataServerConfig { } /// Get allow origins. - pub fn allow_origins(&self) -> &AllowType { + pub fn allow_origins(&self) -> &AllowType { self.cors.allow_origins() } /// Get allow headers. - pub fn allow_headers(&self) -> &AllowType { + pub fn allow_headers(&self) -> &AllowType { self.cors.allow_headers() } /// Get allow methods. - pub fn allow_methods(&self) -> &AllowType { + pub fn allow_methods(&self) -> &AllowType { self.cors.allow_methods() } @@ -304,7 +324,7 @@ impl DataServerConfig { } /// Get the expose headers. - pub fn expose_headers(&self) -> &AllowType { + pub fn expose_headers(&self) -> &AllowType { self.cors.expose_headers() } } @@ -414,20 +434,21 @@ impl Default for Config { impl Config { /// Create a new config. - pub fn new(ticket_server: TicketServerConfig, data_server: Option, resolvers: Vec) -> Self { - Self { - ticket_server, + pub fn new( + ticket_server: TicketServerConfig, + data_server: Option, + resolvers: Vec, + ) -> Self { + Self { + ticket_server, data_server: match data_server { - None => { - DataServerConfigOption::None(DataServerConfigNone::None) - } - Some(value) => { - DataServerConfigOption::Some(value) - } - }, - resolvers } - } - + None => DataServerConfigOption::None(DataServerConfigNone::None), + Some(value) => DataServerConfigOption::Some(value), + }, + resolvers, + } + } + /// Parse the command line arguments pub fn parse_args() -> PathBuf { Args::parse().config.unwrap_or_else(|| "".into()) @@ -550,7 +571,7 @@ mod tests { #[test] fn config_ticket_server_cors_allow_origin_env() { test_config_from_env( - vec![("HTSGET_TICKET_SERVER_ALLOW_CREDENTIALS", true)], + vec![("HTSGET_TICKET_SERVER_CORS_ALLOW_CREDENTIALS", true)], |config| { assert!(config.ticket_server().allow_credentials()); }, @@ -606,7 +627,7 @@ mod tests { #[test] fn config_ticket_server_cors_allow_origin_file() { - test_config_from_file(r#"ticket_server_allow_credentials = true"#, |config| { + test_config_from_file(r#"ticket_server_cors_allow_credentials = true"#, |config| { assert!(config.ticket_server().allow_credentials()); }); } diff --git a/htsget-config/src/lib.rs b/htsget-config/src/lib.rs index 44adba8d4..47076b239 100644 --- a/htsget-config/src/lib.rs +++ b/htsget-config/src/lib.rs @@ -166,13 +166,19 @@ impl Interval { } } +/// Tagged Any allow type for cors config. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub enum TaggedTypeAll { + #[serde(alias = "all", alias = "ALL")] + All, +} + /// Possible values for the fields parameter. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] #[serde(untagged)] pub enum Fields { /// Include all fields - #[serde(alias = "all", alias = "ALL")] - All, + Tagged(TaggedTypeAll), /// List of fields to include List(HashSet), } @@ -182,8 +188,7 @@ pub enum Fields { #[serde(untagged)] pub enum Tags { /// Include all tags - #[serde(alias = "all", alias = "ALL")] - All, + Tagged(TaggedTypeAll), /// List of tags to include List(HashSet), } @@ -216,8 +221,8 @@ impl Query { class: Class::Body, reference_name: None, interval: Interval::default(), - fields: Fields::All, - tags: Tags::All, + fields: Fields::Tagged(TaggedTypeAll::All), + tags: Tags::Tagged(TaggedTypeAll::All), no_tags: NoTags(None), } } diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 5144dd470..4e0ff9242 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -1,13 +1,14 @@ use http::uri::Authority; use regex::{Error, Regex}; use serde::{Deserialize, Serialize}; +use serde_with::with_prefix; +use std::collections::HashSet; use tracing::instrument; use crate::config::{default_localstorage_addr, default_path, default_serve_at}; use crate::regex_resolver::aws::S3Resolver; -use crate::regex_resolver::ReferenceNames::All; use crate::Format::{Bam, Bcf, Cram, Vcf}; -use crate::{Class, Fields, Format, Interval, NoTags, Query, Tags}; +use crate::{Class, Fields, Format, Interval, NoTags, Query, TaggedTypeAll, Tags}; #[cfg(feature = "s3-storage")] pub mod aws; @@ -74,14 +75,14 @@ impl LocalResolver { scheme: Scheme, authority: Authority, local_path: String, - path_prefix: String + path_prefix: String, ) -> Self { - Self { - scheme, - authority, - local_path, - path_prefix - } + Self { + scheme, + authority, + local_path, + path_prefix, + } } /// Get the scheme. @@ -124,29 +125,31 @@ pub struct RegexResolver { regex: Regex, // Todo: should match guard be allowed as variables inside the substitution string? substitution_string: String, - guard: QueryGuard, storage_type: StorageType, + guard: QueryGuard, } +with_prefix!(allow_interval_prefix "allow_interval_"); + /// A query guard represents query parameters that can be allowed to resolver for a given query. #[derive(Serialize, Clone, Debug, Deserialize)] #[serde(default)] pub struct QueryGuard { - allow_formats: Vec, - allow_classes: Vec, allow_reference_names: ReferenceNames, - allow_interval: Interval, allow_fields: Fields, allow_tags: Tags, + allow_formats: Vec, + allow_classes: Vec, + #[serde(flatten, with = "allow_interval_prefix")] + allow_interval: Interval, } /// Reference names that can be matched. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(untagged)] pub enum ReferenceNames { - All, - #[serde(with = "serde_regex")] - Some(Regex), + Tagged(TaggedTypeAll), + List(HashSet), } impl QueryGuard { @@ -160,16 +163,16 @@ impl QueryGuard { &self.allow_classes } - /// Get allow reference names. - pub fn allow_reference_names(&self) -> &ReferenceNames { - &self.allow_reference_names - } - /// Get allow interval. pub fn allow_interval(&self) -> Interval { self.allow_interval } + /// Get allow reference names. + pub fn allow_reference_names(&self) -> &ReferenceNames { + &self.allow_reference_names + } + /// Get allow fields. pub fn allow_fields(&self) -> &Fields { &self.allow_fields @@ -186,10 +189,10 @@ impl Default for QueryGuard { Self { allow_formats: vec![Bam, Cram, Vcf, Bcf], allow_classes: vec![Class::Body, Class::Header], - allow_reference_names: All, allow_interval: Default::default(), - allow_fields: Fields::All, - allow_tags: Tags::All, + allow_reference_names: ReferenceNames::Tagged(TaggedTypeAll::All), + allow_fields: Fields::Tagged(TaggedTypeAll::All), + allow_tags: Tags::Tagged(TaggedTypeAll::All), } } } @@ -197,9 +200,11 @@ impl Default for QueryGuard { impl QueryMatcher for ReferenceNames { fn query_matches(&self, query: &Query) -> bool { match (self, &query.reference_name) { - (ReferenceNames::All, _) => true, - (ReferenceNames::Some(regex), Some(reference_name)) => regex.is_match(reference_name), - (ReferenceNames::Some(_), None) => false, + (ReferenceNames::Tagged(TaggedTypeAll::All), _) => true, + (ReferenceNames::List(reference_names), Some(reference_name)) => { + reference_names.contains(reference_name) + } + (ReferenceNames::List(_), None) => false, } } } @@ -207,11 +212,11 @@ impl QueryMatcher for ReferenceNames { impl QueryMatcher for Fields { fn query_matches(&self, query: &Query) -> bool { match (self, &query.fields) { - (Fields::All, _) => true, + (Fields::Tagged(TaggedTypeAll::All), _) => true, (Fields::List(self_fields), Fields::List(query_fields)) => { self_fields.is_subset(query_fields) } - (Fields::List(_), Fields::All) => false, + (Fields::List(_), Fields::Tagged(TaggedTypeAll::All)) => false, } } } @@ -219,9 +224,9 @@ impl QueryMatcher for Fields { impl QueryMatcher for Tags { fn query_matches(&self, query: &Query) -> bool { match (self, &query.tags) { - (Tags::All, _) => true, + (Tags::Tagged(TaggedTypeAll::All), _) => true, (Tags::List(self_tags), Tags::List(query_tags)) => self_tags.is_subset(query_tags), - (Tags::List(_), Tags::All) => false, + (Tags::List(_), Tags::Tagged(TaggedTypeAll::All)) => false, } } } @@ -230,13 +235,13 @@ impl QueryMatcher for QueryGuard { fn query_matches(&self, query: &Query) -> bool { self.allow_formats.contains(&query.format) && self.allow_classes.contains(&query.class) - && self.allow_reference_names.query_matches(query) && self .allow_interval .contains(query.interval.start.unwrap_or(u32::MIN)) && self .allow_interval .contains(query.interval.end.unwrap_or(u32::MAX)) + && self.allow_reference_names.query_matches(query) && self.allow_fields.query_matches(query) && self.allow_tags.query_matches(query) } @@ -269,7 +274,7 @@ impl RegexResolver { pub fn regex(&self) -> &Regex { &self.regex } - + /// Get the substitution string. pub fn substitution_string(&self) -> &str { &self.substitution_string @@ -295,25 +300,25 @@ impl RegexResolver { self.guard.allow_classes() } - /// Get allow reference names. - pub fn allow_reference_names(&self) -> &ReferenceNames { - &self.guard.allow_reference_names - } - /// Get allow interval. pub fn allow_interval(&self) -> Interval { self.guard.allow_interval } - /// Get allow fields. - pub fn allow_fields(&self) -> &Fields { - &self.guard.allow_fields - } - - /// Get allow tags. - pub fn allow_tags(&self) -> &Tags { - &self.guard.allow_tags - } + // /// Get allow reference names. + // pub fn allow_reference_names(&self) -> &ReferenceNames { + // &self.guard.allow_reference_names + // } + // + // /// Get allow fields. + // pub fn allow_fields(&self) -> &Fields { + // &self.guard.allow_fields + // } + // + // /// Get allow tags. + // pub fn allow_tags(&self) -> &Tags { + // &self.guard.allow_tags + // } } impl Resolver for RegexResolver { diff --git a/htsget-search/src/htsget/mod.rs b/htsget-search/src/htsget/mod.rs index 613d8c309..842142ea7 100644 --- a/htsget-search/src/htsget/mod.rs +++ b/htsget-search/src/htsget/mod.rs @@ -231,7 +231,7 @@ impl Response { #[cfg(test)] mod tests { use super::*; - use htsget_config::{Fields, NoTags, Tags}; + use htsget_config::{Fields, NoTags, TaggedTypeAll, Tags}; use std::collections::HashSet; #[test] @@ -342,8 +342,8 @@ mod tests { #[test] fn query_with_tags() { - let result = Query::new("NA12878", Format::Bam).with_tags(Tags::All); - assert_eq!(result.tags(), &Tags::All); + let result = Query::new("NA12878", Format::Bam).with_tags(Tags::Tagged(TaggedTypeAll::All)); + assert_eq!(result.tags(), &Tags::Tagged(TaggedTypeAll::All)); } #[test] diff --git a/htsget-search/src/storage/mod.rs b/htsget-search/src/storage/mod.rs index 90c97e63a..610973a77 100644 --- a/htsget-search/src/storage/mod.rs +++ b/htsget-search/src/storage/mod.rs @@ -9,7 +9,7 @@ use std::time::Duration; use async_trait::async_trait; use base64::encode; -use htsget_config::config::cors::{AllowType, CorsConfig, TaggedAllowTypes, TaggedAnyAllowType}; +use htsget_config::config::cors::{AllowType, CorsConfig, TaggedAllowTypes}; use htsget_config::regex_resolver::{LocalResolver, Scheme}; use htsget_config::{Class, Query}; use http::{uri, HeaderValue, Method}; diff --git a/htsget-test-utils/src/http_tests.rs b/htsget-test-utils/src/http_tests.rs index 4b352e732..a3651ab18 100644 --- a/htsget-test-utils/src/http_tests.rs +++ b/htsget-test-utils/src/http_tests.rs @@ -4,9 +4,10 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use async_trait::async_trait; -use htsget_config::config::cors::{AllowType, CorsConfig, TaggedAnyAllowType}; +use htsget_config::config::cors::{AllowType, CorsConfig}; use htsget_config::config::{DataServerConfig, TicketServerConfig}; use htsget_config::regex_resolver::{LocalResolver, RegexResolver, Scheme, StorageType}; +use htsget_config::TaggedTypeAll; use http::uri::Authority; use http::HeaderMap; use serde::de; @@ -95,7 +96,7 @@ pub fn default_test_resolver(addr: SocketAddr, scheme: Scheme) -> RegexResolver scheme, Authority::from_str(&addr.to_string()).unwrap(), default_dir_data().to_str().unwrap().to_string(), - "/data".to_string() + "/data".to_string(), ); RegexResolver::new(StorageType::Local(resolver), ".*", "$0", Default::default()).unwrap() @@ -118,14 +119,19 @@ pub fn default_cors_config() -> CorsConfig { CorsConfig::new( false, AllowType::List(vec!["http://example.com".parse().unwrap()]), - AllowType::Tagged(TaggedAnyAllowType::Any), - AllowType::Tagged(TaggedAnyAllowType::Any), + AllowType::Tagged(TaggedTypeAll::All), + AllowType::Tagged(TaggedTypeAll::All), 1000, AllowType::List(vec![]), ) } -fn default_test_config_params(addr: SocketAddr, key: Option, cert: Option, scheme: Scheme) -> Config { +fn default_test_config_params( + addr: SocketAddr, + key: Option, + cert: Option, + scheme: Scheme, +) -> Config { let cors = default_cors_config(); let server_config = DataServerConfig::new( addr, @@ -133,17 +139,13 @@ fn default_test_config_params(addr: SocketAddr, key: Option, cert: Opti PathBuf::from("/data"), key, cert, - cors.clone() + cors.clone(), ); Config::new( - TicketServerConfig::new( - "127.0.0.1:8080".parse().unwrap(), - cors, - Default::default() - ), + TicketServerConfig::new("127.0.0.1:8080".parse().unwrap(), cors, Default::default()), Some(server_config), - vec![default_test_resolver(addr, scheme)] + vec![default_test_resolver(addr, scheme)], ) } From 6b5e60b0d3775f04f1fa13323b0fb9caf0606752 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 21 Dec 2022 14:13:38 +1100 Subject: [PATCH 32/45] config: update config file with default values, add option to print a default config --- htsget-config/config.toml | 95 +++++++++++++++------------------ htsget-config/src/config/mod.rs | 23 ++++++-- htsget-http-actix/src/main.rs | 37 +++++++------ htsget-http-lambda/src/main.rs | 15 ++++-- 4 files changed, 96 insertions(+), 74 deletions(-) diff --git a/htsget-config/config.toml b/htsget-config/config.toml index cd96af9e3..67bb7ceb2 100644 --- a/htsget-config/config.toml +++ b/htsget-config/config.toml @@ -1,51 +1,44 @@ -#ticket_server_addr = '127.0.0.1:8080' -#ticket_server_cors_allow_credentials = false -#ticket_server_cors_allow_origins = ['http://localhost:8080'] -#ticket_server_cors_allow_headers = 'Any' -#ticket_server_cors_allow_methods = 'Any' -#ticket_server_cors_max_age = 86400 -#ticket_server_cors_expose_headers = [] -# -## To disable the data server: -## data_server = '' -#[data_server] -#addr = '127.0.0.1:8081' -#local_path = 'data' -#serve_at = '/data' -#cors_allow_credentials = false -#cors_allow_origins = ['http://localhost:8080'] -#cors_allow_headers = 'Any' -#cors_allow_methods = 'Any' -#cors_max_age = 86400 -#cors_expose_headers = [] -# -#[[resolvers]] -#regex = '.*' -#substitution_string = '$0' -# -#[resolvers.guard] -#allow_formats = [ -# 'BAM', -# 'CRAM', -# 'VCF', -# 'BCF', -#] -#allow_classes = [ -# 'body', -# 'header', -#] -# -##allow_interval.start = 0 -##allow_interval.end = 100 -# -## Default is to allow all reference names, fields, and tags. -##allow_reference_names = ['chr1'] -##allow_fields = ['QNAME'] -##allow_tags = ['RG'] -# -#[resolvers.storage_type] -#type = 'Local' -#scheme = 'Http' -#authority = '127.0.0.1:8081' -#local_path = 'data' -#path_prefix = '/data' +ticket_server_addr = '127.0.0.1:8080' +ticket_server_cors_allow_credentials = false +ticket_server_cors_allow_origins = ['http://localhost:8080'] +ticket_server_cors_allow_headers = 'All' +ticket_server_cors_allow_methods = 'All' +ticket_server_cors_max_age = 86400 +ticket_server_cors_expose_headers = [] + +[data_server] +addr = '127.0.0.1:8081' +local_path = 'data' +serve_at = '/data' +cors_allow_credentials = false +cors_allow_origins = ['http://localhost:8080'] +cors_allow_headers = 'All' +cors_allow_methods = 'All' +cors_max_age = 86400 +cors_expose_headers = [] + +[[resolvers]] +regex = '.*' +substitution_string = '$0' + +[resolvers.storage_type] +type = 'Local' +scheme = 'Http' +authority = '127.0.0.1:8081' +local_path = 'data' +path_prefix = '/data' + +[resolvers.guard] +allow_reference_names = 'All' +allow_fields = 'All' +allow_tags = 'All' +allow_formats = [ + 'BAM', + 'CRAM', + 'VCF', + 'BCF', +] +allow_classes = [ + 'body', + 'header', +] diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index ebe5c8035..7821b98cf 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -87,8 +87,15 @@ pub(crate) fn default_serve_at() -> &'static str { #[derive(Parser, Debug)] #[command(author, version, about, long_about = USAGE)] struct Args { - #[arg(short, long, env = "HTSGET_CONFIG")] + #[arg( + short, + long, + env = "HTSGET_CONFIG", + help = "Set the location of the config file" + )] config: Option, + #[arg(short, long, exclusive = true, help = "Print a default config file")] + print_default_config: bool, } /// Configuration for the htsget server. @@ -450,8 +457,18 @@ impl Config { } /// Parse the command line arguments - pub fn parse_args() -> PathBuf { - Args::parse().config.unwrap_or_else(|| "".into()) + pub fn parse_args() -> Option { + let args = Args::parse(); + + if args.print_default_config { + println!( + "{}", + toml::ser::to_string_pretty(&Config::default()).unwrap() + ); + None + } else { + Some(args.config.unwrap_or_else(|| "".into())) + } } /// Read the environment variables into a Config struct. diff --git a/htsget-http-actix/src/main.rs b/htsget-http-actix/src/main.rs index 9b83b3d66..af18be5d7 100644 --- a/htsget-http-actix/src/main.rs +++ b/htsget-http-actix/src/main.rs @@ -12,24 +12,31 @@ use htsget_search::storage::data_server::HttpTicketFormatter; #[actix_web::main] async fn main() -> std::io::Result<()> { Config::setup_tracing()?; - let config = Config::from_env(Config::parse_args())?; - if let Some(server) = config.data_server() { - let server = server.clone(); - let mut formatter = HttpTicketFormatter::try_from(server.clone())?; - let local_server = formatter.bind_data_server().await?; - let local_server = tokio::spawn(async move { local_server.serve(&server.local_path()).await }); + if let Some(config) = Config::parse_args() { + let config = Config::from_env(config)?; - let ticket_server_config = config.ticket_server().clone(); - select! { - local_server = local_server => Ok(local_server??), - actix_server = run_server( - config.owned_resolvers(), - ticket_server_config, - )? => actix_server + if let Some(server) = config.data_server() { + let server = server.clone(); + let mut formatter = HttpTicketFormatter::try_from(server.clone())?; + + let local_server = formatter.bind_data_server().await?; + let local_server = + tokio::spawn(async move { local_server.serve(&server.local_path()).await }); + + let ticket_server_config = config.ticket_server().clone(); + select! { + local_server = local_server => Ok(local_server??), + actix_server = run_server( + config.owned_resolvers(), + ticket_server_config, + )? => actix_server + } + } else { + let ticket_server_config = config.ticket_server().clone(); + run_server(config.owned_resolvers(), ticket_server_config)?.await } } else { - let ticket_server_config = config.ticket_server().clone(); - run_server(config.owned_resolvers(), ticket_server_config)?.await + Ok(()) } } diff --git a/htsget-http-lambda/src/main.rs b/htsget-http-lambda/src/main.rs index 2df2d3e08..82c704cf0 100644 --- a/htsget-http-lambda/src/main.rs +++ b/htsget-http-lambda/src/main.rs @@ -14,11 +14,16 @@ use htsget_search::storage::local::LocalStorage; #[tokio::main] async fn main() -> Result<(), Error> { Config::setup_tracing()?; - let config = Config::from_env(Config::parse_args())?; - let service_info = config.ticket_server().service_info().clone(); - let cors = config.ticket_server().cors().clone(); - let router = &Router::new(Arc::new(config.owned_resolvers()), &service_info); + if let Some(config) = Config::parse_args() { + let config = Config::from_env(config)?; - handle_request(cors, router).await + let service_info = config.ticket_server().service_info().clone(); + let cors = config.ticket_server().cors().clone(); + let router = &Router::new(Arc::new(config.owned_resolvers()), &service_info); + + handle_request(cors, router).await + } else { + Ok(()) + } } From b1c0ec428250f161311bb97cbe54ab51b78a5a4c Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 21 Dec 2022 14:44:50 +1100 Subject: [PATCH 33/45] style: clippy and fmt --- htsget-config/src/config/mod.rs | 5 +--- htsget-config/src/regex_resolver/mod.rs | 2 +- htsget-http-actix/src/lib.rs | 4 +-- htsget-http-actix/src/main.rs | 7 +---- htsget-http-core/src/lib.rs | 2 -- htsget-http-lambda/src/lib.rs | 14 ++++------ htsget-http-lambda/src/main.rs | 11 ++------ htsget-search/benches/search_benchmarks.rs | 3 --- htsget-search/src/htsget/cram_search.rs | 10 +++---- htsget-search/src/htsget/from_storage.rs | 25 +++++------------ htsget-search/src/htsget/search.rs | 2 +- htsget-search/src/storage/aws.rs | 31 ++++------------------ htsget-search/src/storage/data_server.rs | 1 - htsget-search/src/storage/local.rs | 17 ++---------- htsget-search/src/storage/mod.rs | 16 +++-------- htsget-test-utils/src/server_tests.rs | 3 --- 16 files changed, 34 insertions(+), 119 deletions(-) diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 7821b98cf..9154c411a 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -3,19 +3,16 @@ pub mod cors; use std::fmt::Debug; use std::io; use std::io::ErrorKind; -use std::iter::Map; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAllowTypes}; use clap::Parser; use figment::providers::{Env, Format, Serialized, Toml}; -use figment::value::Value::Dict; use figment::Figment; use http::header::HeaderName; use http::Method; -use serde::de::IntoDeserializer; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde::{Deserialize, Serialize}; use serde_with::with_prefix; use tracing::info; use tracing::instrument; diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 4e0ff9242..8a8e9115f 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -8,7 +8,7 @@ use tracing::instrument; use crate::config::{default_localstorage_addr, default_path, default_serve_at}; use crate::regex_resolver::aws::S3Resolver; use crate::Format::{Bam, Bcf, Cram, Vcf}; -use crate::{Class, Fields, Format, Interval, NoTags, Query, TaggedTypeAll, Tags}; +use crate::{Class, Fields, Format, Interval, Query, TaggedTypeAll, Tags}; #[cfg(feature = "s3-storage")] pub mod aws; diff --git a/htsget-http-actix/src/lib.rs b/htsget-http-actix/src/lib.rs index ce9a24a09..deaa4ab41 100644 --- a/htsget-http-actix/src/lib.rs +++ b/htsget-http-actix/src/lib.rs @@ -7,7 +7,7 @@ use tracing::info; use tracing::instrument; use tracing_actix_web::TracingLogger; -use htsget_config::config::cors::{AllowType, CorsConfig, TaggedAllowTypes}; +use htsget_config::config::cors::CorsConfig; pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig, USAGE}; #[cfg(feature = "s3-storage")] pub use htsget_config::regex_resolver::aws::S3Resolver; @@ -234,7 +234,7 @@ mod tests { async fn get_response( &self, request: test::TestRequest, - formatter: HttpTicketFormatter, + _formatter: HttpTicketFormatter, ) -> ServiceResponse> { let app = test::init_service( App::new() diff --git a/htsget-http-actix/src/main.rs b/htsget-http-actix/src/main.rs index af18be5d7..1d8912688 100644 --- a/htsget-http-actix/src/main.rs +++ b/htsget-http-actix/src/main.rs @@ -1,12 +1,7 @@ -use std::io::{Error, ErrorKind}; - -use htsget_config::config::{DataServerConfig, TicketServerConfig}; -use htsget_config::regex_resolver::RegexResolver; use tokio::select; use htsget_http_actix::run_server; -use htsget_http_actix::{Config, StorageType}; -use htsget_search::htsget::from_storage::HtsGetFromStorage; +use htsget_http_actix::Config; use htsget_search::storage::data_server::HttpTicketFormatter; #[actix_web::main] diff --git a/htsget-http-core/src/lib.rs b/htsget-http-core/src/lib.rs index e13bf4a31..ef6d01f7f 100644 --- a/htsget-http-core/src/lib.rs +++ b/htsget-http-core/src/lib.rs @@ -118,7 +118,6 @@ mod tests { use std::path::PathBuf; use std::sync::Arc; - use htsget_config::regex_resolver::{RegexResolver, StorageType}; use htsget_config::Format; use htsget_search::htsget::HtsGet; use htsget_search::storage::data_server::HttpTicketFormatter; @@ -276,7 +275,6 @@ mod tests { Arc::new(HtsGetFromStorage::new( LocalStorage::new( get_base_path(), - RegexResolver::new(Default::default(), ".*", "$0", Default::default()).unwrap(), HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), CorsConfig::default()), ) .unwrap(), diff --git a/htsget-http-lambda/src/lib.rs b/htsget-http-lambda/src/lib.rs index c77edcd94..826abc8a9 100644 --- a/htsget-http-lambda/src/lib.rs +++ b/htsget-http-lambda/src/lib.rs @@ -4,11 +4,10 @@ use std::collections::HashMap; use std::sync::Arc; -use htsget_config::Class; use lambda_http::ext::RequestExt; use lambda_http::http::{Method, StatusCode, Uri}; -use lambda_http::tower::{ServiceBuilder, ServiceExt}; -use lambda_http::{http, service_fn, Body, Request, Response, Service}; +use lambda_http::tower::ServiceBuilder; +use lambda_http::{http, service_fn, Body, Request, Response}; use lambda_runtime::Error; use tracing::instrument; use tracing::{debug, info}; @@ -205,7 +204,7 @@ mod tests { use std::sync::Arc; use async_trait::async_trait; - use htsget_config::regex_resolver::{LocalResolver, RegexResolver, StorageType}; + use htsget_config::regex_resolver::RegexResolver; use htsget_config::Class; use lambda_http::http::header::HeaderName; use lambda_http::http::Uri; @@ -216,11 +215,8 @@ mod tests { use tempfile::TempDir; use htsget_http_core::Endpoint; - use htsget_search::htsget::from_storage::HtsGetFromStorage; - use htsget_search::htsget::HtsGet; use htsget_search::storage::configure_cors; use htsget_search::storage::data_server::HttpTicketFormatter; - use htsget_search::storage::local::LocalStorage; use htsget_test_utils::http_tests::{config_with_tls, default_test_config, get_test_file}; use htsget_test_utils::http_tests::{Header, Response as TestResponse, TestRequest, TestServer}; use htsget_test_utils::server_tests::{ @@ -297,7 +293,7 @@ mod tests { } async fn test_server(&self, request: LambdaTestRequest) -> TestResponse { - let (expected_path, formatter) = formatter_and_expected_path(self.get_config()).await; + let (expected_path, _formatter) = formatter_and_expected_path(self.get_config()).await; let router = Router::new( Arc::new(self.config.clone().owned_resolvers()), @@ -647,7 +643,7 @@ mod tests { .await; } - async fn with_router<'a, F, Fut>(test: F, config: &'a Config, formatter: HttpTicketFormatter) + async fn with_router<'a, F, Fut>(test: F, config: &'a Config, _formatter: HttpTicketFormatter) where F: FnOnce(Router<'a, Vec>) -> Fut, Fut: Future, diff --git a/htsget-http-lambda/src/main.rs b/htsget-http-lambda/src/main.rs index 82c704cf0..b954648ed 100644 --- a/htsget-http-lambda/src/main.rs +++ b/htsget-http-lambda/src/main.rs @@ -1,15 +1,8 @@ -use std::sync::Arc; - -use htsget_config::config::{DataServerConfig, TicketServerConfig}; -use htsget_config::regex_resolver::RegexResolver; use lambda_http::Error; -use tracing::instrument; +use std::sync::Arc; +use htsget_http_lambda::Config; use htsget_http_lambda::{handle_request, Router}; -use htsget_http_lambda::{Config, StorageType}; -use htsget_search::htsget::from_storage::HtsGetFromStorage; -use htsget_search::storage::data_server::HttpTicketFormatter; -use htsget_search::storage::local::LocalStorage; #[tokio::main] async fn main() -> Result<(), Error> { diff --git a/htsget-search/benches/search_benchmarks.rs b/htsget-search/benches/search_benchmarks.rs index 7b831934b..f5b548154 100644 --- a/htsget-search/benches/search_benchmarks.rs +++ b/htsget-search/benches/search_benchmarks.rs @@ -5,7 +5,6 @@ use criterion::{criterion_group, criterion_main, BenchmarkGroup, Criterion}; use tokio::runtime::Runtime; use htsget_config::config::cors::CorsConfig; -use htsget_config::regex_resolver::StorageType; use htsget_config::Class::Header; use htsget_config::Format::{Bam, Bcf, Cram, Vcf}; use htsget_config::Query; @@ -13,7 +12,6 @@ use htsget_search::htsget::from_storage::HtsGetFromStorage; use htsget_search::htsget::HtsGet; use htsget_search::htsget::HtsGetError; use htsget_search::storage::data_server::HttpTicketFormatter; -use htsget_search::RegexResolver; const BENCHMARK_DURATION_SECONDS: u64 = 15; const NUMBER_OF_SAMPLES: usize = 150; @@ -21,7 +19,6 @@ const NUMBER_OF_SAMPLES: usize = 150; async fn perform_query(query: Query) -> Result<(), HtsGetError> { let htsget = HtsGetFromStorage::local_from( "../data", - RegexResolver::new(StorageType::default(), ".*", "$0", Default::default()).unwrap(), HttpTicketFormatter::new( "127.0.0.1:8081".parse().expect("expected valid address"), CorsConfig::default(), diff --git a/htsget-search/src/htsget/cram_search.rs b/htsget-search/src/htsget/cram_search.rs index 470f56a2d..b5d611eb0 100644 --- a/htsget-search/src/htsget/cram_search.rs +++ b/htsget-search/src/htsget/cram_search.rs @@ -194,7 +194,7 @@ where let owned_record = record.clone(); let owned_next = next.clone(); let owned_predicate = predicate.clone(); - let range = query.interval().clone(); + let range = query.interval(); futures.push_back(tokio::spawn(async move { if owned_predicate(&owned_record) { Self::bytes_ranges_for_record(range, &owned_record, owned_next.offset()) @@ -223,11 +223,9 @@ where )); } Some(last) if predicate(last) => { - if let Some(range) = Self::bytes_ranges_for_record( - query.interval().clone(), - last, - self.position_at_eof(query).await?, - )? { + if let Some(range) = + Self::bytes_ranges_for_record(query.interval(), last, self.position_at_eof(query).await?)? + { byte_ranges.push(range); } } diff --git a/htsget-search/src/htsget/from_storage.rs b/htsget-search/src/htsget/from_storage.rs index 14039bad9..a3386157f 100644 --- a/htsget-search/src/htsget/from_storage.rs +++ b/htsget-search/src/htsget/from_storage.rs @@ -14,9 +14,8 @@ use crate::htsget::search::Search; use crate::htsget::{Format, HtsGetError}; #[cfg(feature = "s3-storage")] use crate::storage::aws::AwsS3Storage; -use crate::storage::data_server::HttpTicketFormatter; use crate::storage::local::LocalStorage; -use crate::storage::{StorageError, UrlFormatter}; +use crate::storage::UrlFormatter; use crate::RegexResolver; use crate::{ htsget::bam_search::BamSearch, @@ -47,14 +46,12 @@ impl HtsGet for &[RegexResolver] { if let Some(id) = resolver.resolve_id(&query) { match resolver.storage_type() { StorageType::Local(url) => { - let searcher = - HtsGetFromStorage::local_from(url.local_path(), resolver.clone(), url.clone())?; + let searcher = HtsGetFromStorage::local_from(url.local_path(), url.clone())?; return searcher.search(query.with_id(id)).await; } #[cfg(feature = "s3-storage")] StorageType::S3(s3) => { - let searcher = - HtsGetFromStorage::s3_from(s3.bucket().to_string(), resolver.clone()).await; + let searcher = HtsGetFromStorage::s3_from(s3.bucket().to_string()).await; return searcher.search(query.with_id(id)).await; } _ => {} @@ -100,20 +97,14 @@ impl HtsGetFromStorage { #[cfg(feature = "s3-storage")] impl HtsGetFromStorage { - pub async fn s3_from(bucket: String, resolver: RegexResolver) -> Self { - HtsGetFromStorage::new(AwsS3Storage::new_with_default_config(bucket, resolver).await) + pub async fn s3_from(bucket: String) -> Self { + HtsGetFromStorage::new(AwsS3Storage::new_with_default_config(bucket).await) } } impl HtsGetFromStorage> { - pub fn local_from>( - path: P, - resolver: RegexResolver, - formatter: T, - ) -> Result { - Ok(HtsGetFromStorage::new(LocalStorage::new( - path, resolver, formatter, - )?)) + pub fn local_from>(path: P, formatter: T) -> Result { + Ok(HtsGetFromStorage::new(LocalStorage::new(path, formatter)?)) } } @@ -126,7 +117,6 @@ pub(crate) mod tests { use htsget_config::config::cors::CorsConfig; use tempfile::TempDir; - use htsget_config::regex_resolver::StorageType; use htsget_test_utils::util::expected_bgzf_eof_data_url; use crate::htsget::bam_search::tests::{ @@ -205,7 +195,6 @@ pub(crate) mod tests { test(Arc::new( LocalStorage::new( base_path, - RegexResolver::new(Default::default(), ".*", "$0", Default::default()).unwrap(), HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), CorsConfig::default()), ) .unwrap(), diff --git a/htsget-search/src/htsget/search.rs b/htsget-search/src/htsget/search.rs index bf07eeab6..9eb7d30d7 100644 --- a/htsget-search/src/htsget/search.rs +++ b/htsget-search/src/htsget/search.rs @@ -405,7 +405,7 @@ where let chunks: Result> = trace_span!("querying chunks").in_scope(|| { trace!(id = ?query.id(), ref_seq_id = ?ref_seq_id, "querying chunks"); let mut chunks = index - .query(ref_seq_id, query.interval().clone().into_one_based()?) + .query(ref_seq_id, query.interval().into_one_based()?) .map_err(|err| HtsGetError::InvalidRange(format!("querying range: {}", err)))?; if chunks.is_empty() { diff --git a/htsget-search/src/storage/aws.rs b/htsget-search/src/storage/aws.rs index 92974d40f..4656d16d5 100644 --- a/htsget-search/src/storage/aws.rs +++ b/htsget-search/src/storage/aws.rs @@ -17,14 +17,11 @@ use tokio_util::io::StreamReader; use tracing::debug; use tracing::instrument; -use htsget_config::Query; - use crate::htsget::Url; use crate::storage::aws::Retrieval::{Delayed, Immediate}; use crate::storage::StorageError::AwsS3Error; use crate::storage::{BytesPosition, StorageError}; use crate::storage::{BytesRange, Storage}; -use crate::RegexResolver; use super::{GetOptions, RangeUrlOptions, Result}; @@ -42,27 +39,18 @@ pub enum Retrieval { pub struct AwsS3Storage { client: Client, bucket: String, - id_resolver: RegexResolver, } impl AwsS3Storage { // Allow the user to set this? pub const PRESIGNED_REQUEST_EXPIRY: u64 = 1000; - pub fn new(client: Client, bucket: String, id_resolver: RegexResolver) -> Self { - AwsS3Storage { - client, - bucket, - id_resolver, - } + pub fn new(client: Client, bucket: String) -> Self { + AwsS3Storage { client, bucket } } - pub async fn new_with_default_config(bucket: String, id_resolver: RegexResolver) -> Self { - AwsS3Storage::new( - Client::new(&aws_config::load_from_env().await), - bucket, - id_resolver, - ) + pub async fn new_with_default_config(bucket: String) -> Self { + AwsS3Storage::new(Client::new(&aws_config::load_from_env().await), bucket) } pub async fn s3_presign_url + Send>( @@ -245,16 +233,11 @@ mod tests { use s3_server::storages::fs::FileSystem; use s3_server::{S3Service, SimpleAuth}; - use htsget_config::regex_resolver::LocalResolver; - use htsget_config::Format::Bam; - use htsget_config::Query; - use crate::htsget::Headers; use crate::storage::aws::AwsS3Storage; use crate::storage::local::tests::create_local_test_files; use crate::storage::StorageError; use crate::storage::{BytesPosition, GetOptions, RangeUrlOptions, Storage}; - use crate::RegexResolver; async fn with_s3_test_server(server_base_path: &Path, test: F) where @@ -300,11 +283,7 @@ mod tests { { let (folder_name, base_path) = create_local_test_files().await; with_s3_test_server(base_path.path(), |client| async move { - test(AwsS3Storage::new( - client, - folder_name, - RegexResolver::new(Default::default(), ".*", "$0", Default::default()).unwrap(), - )); + test(AwsS3Storage::new(client, folder_name)); }) .await; } diff --git a/htsget-search/src/storage/data_server.rs b/htsget-search/src/storage/data_server.rs index 1db01af00..418b7fac2 100644 --- a/htsget-search/src/storage/data_server.rs +++ b/htsget-search/src/storage/data_server.rs @@ -17,7 +17,6 @@ use axum_extra::routing::SpaRouter; use futures_util::future::poll_fn; use htsget_config::config::cors::CorsConfig; use htsget_config::config::DataServerConfig; -use htsget_config::regex_resolver::LocalResolver; use http::uri::Scheme; use hyper::server::accept::Accept; use hyper::server::conn::{AddrIncoming, Http}; diff --git a/htsget-search/src/storage/local.rs b/htsget-search/src/storage/local.rs index 10276e40b..fd74227ed 100644 --- a/htsget-search/src/storage/local.rs +++ b/htsget-search/src/storage/local.rs @@ -9,11 +9,8 @@ use tokio::fs::File; use tracing::debug; use tracing::instrument; -use htsget_config::Query; - use crate::htsget::Url; -use crate::storage::{resolve_id, Storage, UrlFormatter}; -use crate::RegexResolver; +use crate::storage::{Storage, UrlFormatter}; use super::{GetOptions, RangeUrlOptions, Result, StorageError}; @@ -22,16 +19,11 @@ use super::{GetOptions, RangeUrlOptions, Result, StorageError}; #[derive(Debug, Clone)] pub struct LocalStorage { base_path: PathBuf, - id_resolver: RegexResolver, url_formatter: T, } impl LocalStorage { - pub fn new>( - base_path: P, - id_resolver: RegexResolver, - url_formatter: T, - ) -> Result { + pub fn new>(base_path: P, url_formatter: T) -> Result { base_path .as_ref() .to_path_buf() @@ -39,7 +31,6 @@ impl LocalStorage { .map_err(|_| StorageError::KeyNotFound(base_path.as_ref().to_string_lossy().to_string())) .map(|canonicalized_base_path| Self { base_path: canonicalized_base_path, - id_resolver, url_formatter, }) } @@ -132,9 +123,6 @@ pub(crate) mod tests { use tokio::fs::{create_dir, File}; use tokio::io::AsyncWriteExt; - use htsget_config::regex_resolver::StorageType; - use htsget_config::Format::Bam; - use crate::htsget::{Headers, Url}; use crate::storage::data_server::HttpTicketFormatter; use crate::storage::{BytesPosition, GetOptions, RangeUrlOptions, StorageError}; @@ -302,7 +290,6 @@ pub(crate) mod tests { test( LocalStorage::new( base_path.path(), - RegexResolver::new(StorageType::default(), ".*", "$0", Default::default()).unwrap(), HttpTicketFormatter::new("127.0.0.1:8081".parse().unwrap(), CorsConfig::default()), ) .unwrap(), diff --git a/htsget-search/src/storage/mod.rs b/htsget-search/src/storage/mod.rs index 610973a77..89dee306f 100644 --- a/htsget-search/src/storage/mod.rs +++ b/htsget-search/src/storage/mod.rs @@ -9,19 +9,16 @@ use std::time::Duration; use async_trait::async_trait; use base64::encode; -use htsget_config::config::cors::{AllowType, CorsConfig, TaggedAllowTypes}; +use htsget_config::config::cors::CorsConfig; use htsget_config::regex_resolver::{LocalResolver, Scheme}; -use htsget_config::{Class, Query}; -use http::{uri, HeaderValue, Method}; +use htsget_config::Class; +use http::{uri, HeaderValue}; use thiserror::Error; use tokio::io::AsyncRead; use tower_http::cors::{AllowHeaders, AllowMethods, AllowOrigin, CorsLayer, ExposeHeaders}; use tracing::instrument; use crate::htsget::{Headers, Url}; -use crate::storage::data_server::CORS_MAX_AGE; -use crate::storage::StorageError::DataServerError; -use crate::{RegexResolver, Resolver}; #[cfg(feature = "s3-storage")] pub mod aws; @@ -424,13 +421,6 @@ impl RangeUrlOptions { } } -/// Resolve a key id with the `RegexResolver` and convert it to a Result. -fn resolve_id(resolver: &RegexResolver, query: &Query) -> Result { - resolver - .resolve_id(query) - .ok_or_else(|| StorageError::InvalidKey(query.id().to_string())) -} - #[cfg(test)] mod tests { use std::collections::HashMap; diff --git a/htsget-test-utils/src/server_tests.rs b/htsget-test-utils/src/server_tests.rs index 073987d71..19668cf0c 100644 --- a/htsget-test-utils/src/server_tests.rs +++ b/htsget-test-utils/src/server_tests.rs @@ -1,16 +1,13 @@ use std::collections::HashMap; use std::path::PathBuf; -use std::time::Duration; use futures::future::join_all; use futures::TryStreamExt; -use htsget_config::regex_resolver::LocalResolver; use htsget_config::{Class, Format}; use http::Method; use noodles_bgzf as bgzf; use noodles_vcf as vcf; use reqwest::ClientBuilder; -use tokio::time::sleep; use htsget_http_core::{get_service_info_with, Endpoint}; use htsget_search::htsget::Response as HtsgetResponse; From 4293a847d1e5641dbb1bd69ebce64c503332bd97 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 21 Dec 2022 20:28:23 +1100 Subject: [PATCH 34/45] config: flatten data server config --- htsget-config/src/config/mod.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 9154c411a..91b4a01e2 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -95,12 +95,16 @@ struct Args { print_default_config: bool, } +with_prefix!(ticket_server_prefix "ticket_server_"); +with_prefix!(data_server_prefix "data_server_"); + /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct Config { - #[serde(flatten)] + #[serde(flatten, with = "ticket_server_prefix")] ticket_server: TicketServerConfig, + #[serde(flatten, with = "data_server_prefix")] data_server: DataServerConfigOption, resolvers: Vec, } @@ -118,14 +122,14 @@ enum DataServerConfigOption { Some(DataServerConfig), } -with_prefix!(ticket_server_cors_prefix "ticket_server_cors_"); +with_prefix!(cors_prefix "cors_"); /// Configuration for the htsget ticket server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { - ticket_server_addr: SocketAddr, - #[serde(flatten, with = "ticket_server_cors_prefix")] + addr: SocketAddr, + #[serde(flatten, with = "cors_prefix")] cors: CorsConfig, #[serde(flatten)] service_info: ServiceInfo, @@ -135,7 +139,7 @@ impl TicketServerConfig { /// Create a new ticket server config. pub fn new(ticket_server_addr: SocketAddr, cors: CorsConfig, service_info: ServiceInfo) -> Self { Self { - ticket_server_addr, + addr: ticket_server_addr, cors, service_info, } @@ -143,7 +147,7 @@ impl TicketServerConfig { /// Get the addr. pub fn addr(&self) -> SocketAddr { - self.ticket_server_addr + self.addr } /// Get cors config. @@ -237,8 +241,6 @@ impl TicketServerConfig { } } -with_prefix!(cors_prefix "cors_"); - /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] @@ -419,7 +421,7 @@ impl ServiceInfo { impl Default for TicketServerConfig { fn default() -> Self { Self { - ticket_server_addr: default_addr().parse().expect("expected valid address"), + addr: default_addr().parse().expect("expected valid address"), cors: CorsConfig::default(), service_info: ServiceInfo::default(), } From cc87548da90ed8a64c28c8899be6561bcc96bf4c Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 22 Dec 2022 08:36:11 +1100 Subject: [PATCH 35/45] docs: add documentation for reworked config --- htsget-config/README.md | 346 ++++++++++++++++++++++++ htsget-config/src/config/mod.rs | 36 ++- htsget-config/src/regex_resolver/mod.rs | 55 ++-- 3 files changed, 401 insertions(+), 36 deletions(-) create mode 100644 htsget-config/README.md diff --git a/htsget-config/README.md b/htsget-config/README.md new file mode 100644 index 000000000..65df42d2d --- /dev/null +++ b/htsget-config/README.md @@ -0,0 +1,346 @@ +# htsget-config + +[![MIT licensed][mit-badge]][mit-url] +[![Build Status][actions-badge]][actions-url] + +[mit-badge]: https://img.shields.io/badge/license-MIT-blue.svg +[mit-url]: https://github.com/umccr/htsget-rs/blob/main/LICENSE +[actions-badge]: https://github.com/umccr/htsget-rs/actions/workflows/action.yml/badge.svg +[actions-url]: https://github.com/umccr/htsget-rs/actions?query=workflow%3Atests+branch%3Amain + +Configuration for [htsget-rs] and relevant crates. + +[htsget-rs]: https://github.com/umccr/htsget-rs + +## Overview + +This crate is used to configure htsget-rs by using a config file or reading environment variables. + +## Usage + +### For running htsget-rs as an application + +To configure htsget-rs, a TOML config file can be used. It also supports reading config from environment variables. +Any config options set by environment variables override values in the config file. For some of +the more deeply nested config options, it may be more ergonomic to use a config file rather than environment variables. + +The configuration consists of multiple parts, config for the ticket server, config for the data server, service-info config, and config for the resolvers. + +#### Ticket server config +The ticket server responds to htsget requests by returning a set of URL tickets that the client must fetch and concatenate. +To configure the ticket server, set the following options: + +| Config File | Description | Type | Default | +|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------|-----------------------------| +| `ticket_server_addr` | The address for the ticket server. | Socket address | `'127.0.0.1:8080'` | +| `ticket_server_cors_allow_credentials` | Controls the CORS Access-Control-Allow-Credentials for the ticket server. | Boolean | `false` | +| `ticket_server_cors_allow_origins` | Set the CORS Access-Control-Allow-Origin returned by the ticket server, this can be set to `All` to send a wildcard, `Mirror` to echo back the request sent by the client, or a specific array of origins. | `'All'`, `'Mirror'` or a array of origins | `['http://localhost:8080']` | +| `ticket_server_cors_allow_headers` | Set the CORS Access-Control-Allow-Headers returned by the ticket server, this can be set to `All` to allow all headers, or a specific array of headers. | `'All'`, or a array of headers | `'All'` | +| `ticket_server_cors_allow_methods` | Set the CORS Access-Control-Allow-Methods returned by the ticket server, this can be set to `All` to allow all methods, or a specific array of methods. | `'All'`, or a array of methods | `'All'` | +| `ticket_server_cors_max_age` | Set the CORS Access-Control-Max-Age for the ticket server which controls how long a preflight request can be cached for. | Seconds | `86400` | +| `ticket_server_cors_expose_headers` | Set the CORS Access-Control-Expose-Headers returned by the ticket server, this can be set to `All` to expose all headers, or a specific array of headers. | `'All'`, or a array of headers | `[]` | + +An example of config for the ticket server: +```toml +ticket_server_addr = '127.0.0.1:8080' +ticket_server_cors_allow_credentials = false +ticket_server_cors_allow_origins = 'Mirror' +ticket_server_cors_allow_headers = ['Content-Type'] +ticket_server_cors_allow_methods = ['GET', 'POST'] +ticket_server_cors_max_age = 86400 +ticket_server_cors_expose_headers = [] +``` + +#### Local data server config +The local data server responds to tickets produced by the ticket server by serving local filesystem data. +To configure the data server, set the following options: + +| Option | Description | Type | Default | +|-------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------|-----------------------------| +| `data_server_addr` | The address for the data server. | Socket address | `'127.0.0.1:8080'` | +| `data_server_local_path` | The local path which the data server can access to serve files. | Filesystem path | `'data'` | +| `data_server_serve_at` | The path which the data server will prefix to all response URLs for tickets. | URL path | `'/data'` | +| `data_server_key` | The path to the PEM formatted X.509 private key used by the data server. This is used to enable TLS with HTTPS. | Filesystem path | Not set | +| `data_server_cert` | The path to the PEM formatted X.509 certificate used by the data server. This is used to enable TLS with HTTPS. | Filesystem path | Not set | +| `data_server_cors_allow_credentials` | Controls the CORS Access-Control-Allow-Credentials for the data server. | Boolean | `false` | +| `data_server_cors_allow_origins` | Set the CORS Access-Control-Allow-Origin returned by the data server, this can be set to `All` to send a wildcard, `Mirror` to echo back the request sent by the client, or a specific array of origins. | `'All'`, `'Mirror'` or a array of origins | `['http://localhost:8080']` | +| `data_server_cors_allow_headers` | Set the CORS Access-Control-Allow-Headers returned by the data server, this can be set to `All` to allow all headers, or a specific array of headers. | `'All'`, or a array of headers | `'All'` | +| `data_server_cors_allow_methods` | Set the CORS Access-Control-Allow-Methods returned by the data server, this can be set to `All` to allow all methods, or a specific array of methods. | `'All'`, or a array of methods | `'All'` | +| `data_server_cors_max_age` | Set the CORS Access-Control-Max-Age for the data server which controls how long a preflight request can be cached for. | Seconds | `86400` | +| `data_server_cors_expose_headers` | Set the CORS Access-Control-Expose-Headers returned by the data server, this can be set to `All` to expose all headers, or a specific array of headers. | `'All'`, or a array of headers | `[]` | + +An example of config for the data server: +```toml +data_server_addr = '127.0.0.1:8081' +data_server_local_path = 'data' +data_server_serve_at = '/data' +data_server_key = 'key.pem' +data_server_cert = 'cert.pem' +data_server_cors_allow_credentials = false +data_server_cors_allow_origins = 'Mirror' +data_server_cors_allow_headers = ['Content-Type'] +data_server_cors_allow_methods = ['GET', 'POST'] +data_server_cors_max_age = 86400 +data_server_cors_expose_headers = [] +``` + +Sometimes it may be useful to disable the data server as all responses to the ticket server will be handled elsewhere, such as with an AWS S3 data server. + +To disable the data server, set the following option: + +
+data_server = ''
+
+ +#### Service info config + +The service info config controls what is returned when the [`service-info`][service-info] path is queried.
+To configure the service-info, set the following options: + +| Option | Description | Type | Default | +|---------------------------------------------------------|---------------------------------------------|-----------|----------| +| `id` | Service ID. | String | Not set | +| `name` | Service name. | String | Not set | +| `version` | Service version. | String | Not set | +| `organization_name` | Organization name. | String | Not set | +| `organization_url` | Organization URL. | String | Not set | +| `contact_url` | Service contact URL | String | Not set | +| `documentation_url` | Service documentation URL. | String | Not set | +| `created_at` | When the service was created. | String | Not set | +| `updated_at` | When the service was last updated. | String | Not set | +| `environment` | The environment the service is running in. | String | Not set | + +An example of config for the service info: +```toml +id = 'id' +name = 'name' +version = '0.1' +organization_name = 'name' +organization_url = 'https://example.com/' +contact_url = 'mailto:nobody@example.com' +documentation_url = 'https://example.com/' +created_at = '2022-01-01T12:00:00Z' +updated_at = '2022-01-01T12:00:00Z' +environment = 'dev' +``` + +#### Resolvers + +The resolvers component of htsget-rs is used to map query IDs to the location of the resource. Each query that htsget-rs receives is +'resolved' to a location, which a data server can respond with. A query ID is matched with a regex, and is then mapped with a substitution string that +has access to the regex capture groups. Each resolver is an array of TOML of tables that attempts to match a query ID. This array matches IDs in order, meaning that +the first matching resolver is resolver used to map the ID. + +To create a resolver, add a `[[resolvers]]` array of tables, and set the following options: + +| Option | Description | Type | Default | +|-----------------------|-------------------------------------------------------------------------------------------------------------------------|---------------------------------------|---------| +| `regex` | A regular expression which can match a query ID. | Regex | '.*' | +| `substitution_string` | The replacement expression used to map the matched query ID. This has access to the match groups in the `regex` option. | String with access to capture groups | '$0' | + +For example, below is a `regex` option which matches a `/` between two groups, and inserts an additional `data` +inbetween the groups with the `substitution_string`. + +```toml +regex = '(?P.*?)/(?P.*)' +substitution = '$group1/data/$group2' +``` + +For more information about regex options see the [regex crate](https://docs.rs/regex/). + +Each resolver also maps to a certain storage type. This storage type can be used to set query IDs which are served from local storage, or on AWS S3. +To set the storage type for a resolver, add a `[resolvers.storage_type]` table. Set the type option to control the data server storage type: + +| Option | Description | Type | Default | +|---------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|---------------------| +| `type` | The storage type. | Either `'Local'` or `'S3'` | `'Local'` | + +If the type is `Local`, then the following options can be set: + +| Option | Description | Type | Default | +|---------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|---------------------| +| `scheme` | The scheme present on URL tickets. | Either `'HTTP'` or `'HTTPS'` | `'HTTP'` | +| `authority` | The authority present on URL tickets. This should likely match the `data_server_addr`. | URL authority | `'127.0.0.1:8081' ` | +| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server_local_path`. | Filesystem path | `'data'` | +| `path_prefix` | The path prefix which the URL tickets will have. This should likely match the `data_server_serve_at` path. | URL path | `'/data'` | + +If the type is `S3`, then the following option can be set: + +| Option | Description | Type | Default | +|----------|----------------------------------------------------------|-----------------------------|----------| +| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | Not set | + +Additionally, the resolver component has a feature, which allows resolving IDs based on the other fields present in a query. +This is useful as allows the resolver to match only match an ID, if a particular set of query parameters are also present. For example, +a resolver can be set to only resolve IDs if the format is also BAM. + +To set the resolver 'allow guard', add a `[resolver.allow_guard]` table, and set the following options: + +| Option | Description | Type | Default | +|-------------------------|-----------------------------------------------------------------------------------------|-----------------------------------------------------------------------|-------------------------------------| +| `allow_reference_names` | Resolve the query ID if the query also contains the reference names set by this option. | Array of reference names or `'All'` | `'All'` | +| `allow_fields` | Resolve the query ID if the query also contains the fields set by this option. | Array of fields or `'All'` | `'All'` | +| `allow_tags` | Resolve the query ID if the query also contains the tags set by this option. | Array of tags or `'All'` | `'All'` | +| `allow_formats` | Resolve the query ID if the query is one of the formats specified by this option. | An array of formats containing `'BAM'`, `'CRAM'`, `'VCF'`, or `'BCF'` | `['BAM', 'CRAM', 'VCF', 'BCF']` | +| `allow_classes` | Resolve the query ID if the query is one of the classes specified by this option. | An array of classes containing eithr `'body'` or `'header'` | `['body', 'header']` | +| `allow_interval_start` | Resolve the query ID if the query reference start position is at least this option. | Unsigned 32-bit integer start position, 0-based, inclusive | Not set, allows all start positions | +| `allow_interval_end` | Resolve the query ID if the query reference end position is at most this option. | Unsigned 32-bit integer end position, 0-based exclusive. | Not set, allows all end positions | + +An example of a fully configured resolver: + +```toml +[[resolvers]] +regex = '.*' +substitution_string = '$0' + +[resolvers.storage_type] +type = 'S3' +bucket = 'bucket' + +[resolvers.allow_guard] +allow_reference_names = ['chr1'] +allow_fields = ['QNAME'] +allow_tags = ['RG'] +allow_formats = ['BAM'] +allow_classes = ['body'] +allow_interval_start = 100 +allow_interval_end = 1000 +``` + +#### Config file location + +The htsget-rs binaries ([htsget-http-actix] and [htsget-http-lambda]) support some command line options. The config file location can +be specified by setting the `--config` option: + +```shell +cargo run -p htsget-http-actix -- --config "config.toml" +``` + +The config can also be read from an environment variable: + +```shell +export HTSGET_CONFIG="config.toml" +``` +If no config file is specified, the default configuration is used. Further, the default configuration file can be printed to stdout by passing +the `--print-default-config` flag: + +```shell +cargo run -p htsget-http-actix -- --print-default-config +``` + +Pass the `--help` flag to see more details on command line options: + +[htsget-http-actix]: ../htsget-http-actix +[htsget-http-lambda]: ../htsget-http-lambda + +#### Configuring htsget-rs with environment variables + +All the htsget-rs config options can be set by environment variables. The ticket server, data server and service info options are flattened and can be set directly using +environment variable. It is not recommended to set the resolvers using environment variables, however it can be done by setting a single environment variable which +contains a list of structures, where a key name and value pair is used to set the nested options. + +Environment variables will override options set in the config file. Note, arrays are delimited with `[` and `]` in environment variables, and items are separated by commas. + +The following environment variables - corresponding to the TOML config - are available: + +| Variable | Description | Default | +|---------------------------------------------|-------------------------------------------------------------------------------------|-------------------------| +| HTSGET_TICKET_SERVER_ADDR | See [`ticket_server_addr`](#ticket_server_addr) | "data" | +| HTSGET_TICKET_SERVER_CORS_ALLOW_CREDENTIALS | See [`ticket_server_cors_allow_credentials`](#ticket_server_cors_allow_credentials) | ".*" | +| HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGINS | See [`ticket_server_cors_allow_origins`](#ticket_server_cors_allow_origins) | "$0" | +| HTSGET_TICKET_SERVER_CORS_ALLOW_HEADERS | See [`ticket_server_cors_allow_headers`](#ticket_server_cors_allow_headers) | "LocalStorage" | +| HTSGET_TICKET_SERVER_CORS_MAX_AGE | See [`ticket_server_cors_max_age`](#ticket_server_cors_max_age) | "127.0.0.1:8080" | +| HTSGET_TICKET_SERVER_CORS_EXPOSE_HEADERS | See [`ticket_server_cors_expose_headers`](#ticket_server_cors_expose_headers) | "false" | +| HTSGET_DATA_SERVER_ADDR | See [`data_server_addr`](#data_server_addr) | "127.0.0.1:8081" | +| HTSGET_DATA_SERVER_LOCAL_PATH | See [`data_server_local_path`](#data_server_local_path) | "None" | +| HTSGET_DATA_SERVER_SERVE_AT | See [`data_server_serve_at`](#data_server_serve_at) | "None" | +| HTSGET_DATA_SERVER_CORS_ALLOW_CREDENTIALS | See [`data_server_cors_allow_credentials`](#data_server_cors_allow_credentials) | "false" | +| HTSGET_DATA_SERVER_CORS_ALLOW_ORIGINS | See [`data_server_cors_allow_origins`](#data_server_cors_allow_origins) | "http://localhost:8081" | +| HTSGET_DATA_SERVER_CORS_ALLOW_HEADERS | See [`data_server_cors_allow_headers`](#data_server_cors_allow_headers) | "" | +| HTSGET_DATA_SERVER_CORS_MAX_AGE | See [`data_server_cors_max_age`](#data_server_cors_max_age) | | +| HTSGET_DATA_SERVER_CORS_EXPOSE_HEADERS | See [`data_server_cors_expose_headers`](#data_server_cors_expose_headers) | | +| HTSGET_ID | See [`id`](#id) | "None" | +| HTSGET_NAME | See [`name`](#name) | "None" | +| HTSGET_VERSION | See [`version`](#version) | "None" | +| HTSGET_ORGANIZATION_NAME | See [`organization_name`](#organization_name) | "None" | +| HTSGET_ORGANIZATION_URL | See [`organization_url`](#organization_url) | "None" | +| HTSGET_CONTACT_URL | See [`contact_url`](#contact_url) | "None" | +| HTSGET_DOCUMENTATION_URL | See [`documentation_url`](#documentation_url) | "None" | +| HTSGET_CREATED_AT | See [`created_at`](#created_at) | "None" | +| HTSGET_UPDATED_AT | See [`updated_at`](#updated_at) | "None" | +| HTSGET_ENVIRONMENT | See [`environment`](#environment) | "None" | +| HTSGET_RESOLVERS | See [resolvers](#resolvers) | | + +In order to use `HTSGET_RESOLVERS`, the entire resolver config array must be set. The nested array of resolvers structure can be set using name key and value pairs, for example: + +```shell +export HTSGET_RESOLVERS="[{ + regex="regex", + substitution_string="substitution_string", + storage_type={ + type="S3", + bucket="bucket" + }, + allow_guard={ + allow_reference_names="['chr1']", + allow_fields="['QNAME']", + allow_tags="['RG']", + allow_formats="['BAM']", + allow_classes="['body']", + allow_interval_start=100, + allow_interval_end=1000 + } +}]" +``` + +Note the use of double quotes for certin values. + +Similar to the [data_server](#data_server) option, the data server can be disabled by setting the equivalent environment variable: + +```shell +export HTSGET_DATA_SERVER="" +``` +[service-info]: https://samtools.github.io/hts-specs/htsget.html#ga4gh-service-info + +#### RUST_LOG + +The [Tracing][tracing] crate is used extensively by htsget-rs is for logging functionality. The `RUST_LOG` variable is +read to configure the level that trace logs are emitted. + +For example, the following indicates trace level for all htsget crates, and info level for all other crates: + +```sh +export RUST_LOG='info,htsget_http_lambda=trace,htsget_http_lambda=trace,htsget_config=trace,htsget_http_core=trace,htsget_search=trace,htsget_test_utils=trace' +``` + +See [here][rust-log] for more information on setting this variable. + +[tracing]: https://github.com/tokio-rs/tracing +[rust-log]: https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html + +#### AWS config + +Config for AWS is read entirely from environment variables. A default configuration is loaded from environment variables using the [aws-config] crate. +Check out the [AWS documentation][aws-sdk] for the rust SDK for more information. + +[aws-config]: https://docs.rs/aws-config/latest/aws_config/ +[aws-sdk]: https://docs.aws.amazon.com/sdk-for-rust/latest/dg/welcome.html + +### As a library + +This crate reads config files and environment variables using [figment], and accepts command-line arguments using clap. The main function for this is `from_config`, +which is used to obtain the `Config` struct. The crate also contains the `regex_resolver` abstraction, which is used for matching a query ID with +regex, and changing it by using a substitution string. + +[figment]: https://github.com/SergioBenitez/Figment + +#### Feature flags + +This crate has the following features: +* `s3-storage`: used to enable `AwsS3Storage` functionality. + +## License + +This project is licensed under the [MIT license][license]. + +[license]: LICENSE \ No newline at end of file diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 91b4a01e2..0bea686ce 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -95,14 +95,13 @@ struct Args { print_default_config: bool, } -with_prefix!(ticket_server_prefix "ticket_server_"); with_prefix!(data_server_prefix "data_server_"); /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct Config { - #[serde(flatten, with = "ticket_server_prefix")] + #[serde(flatten)] ticket_server: TicketServerConfig, #[serde(flatten, with = "data_server_prefix")] data_server: DataServerConfigOption, @@ -122,14 +121,14 @@ enum DataServerConfigOption { Some(DataServerConfig), } -with_prefix!(cors_prefix "cors_"); +with_prefix!(ticket_server_cors_prefix "ticket_server_cors_"); /// Configuration for the htsget ticket server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct TicketServerConfig { - addr: SocketAddr, - #[serde(flatten, with = "cors_prefix")] + ticket_server_addr: SocketAddr, + #[serde(flatten, with = "ticket_server_cors_prefix")] cors: CorsConfig, #[serde(flatten)] service_info: ServiceInfo, @@ -139,7 +138,7 @@ impl TicketServerConfig { /// Create a new ticket server config. pub fn new(ticket_server_addr: SocketAddr, cors: CorsConfig, service_info: ServiceInfo) -> Self { Self { - addr: ticket_server_addr, + ticket_server_addr, cors, service_info, } @@ -147,7 +146,7 @@ impl TicketServerConfig { /// Get the addr. pub fn addr(&self) -> SocketAddr { - self.addr + self.ticket_server_addr } /// Get cors config. @@ -241,6 +240,8 @@ impl TicketServerConfig { } } +with_prefix!(cors_prefix "cors_"); + /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] @@ -351,7 +352,7 @@ impl Default for DataServerConfig { } /// Configuration of the service info. -#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct ServiceInfo { id: Option, @@ -366,6 +367,23 @@ pub struct ServiceInfo { environment: Option, } +impl Default for ServiceInfo { + fn default() -> Self { + Self { + id: Some("None".to_string()), + name: Some("None".to_string()), + version: Some("None".to_string()), + organization_name: Some("None".to_string()), + organization_url: Some("None".to_string()), + contact_url: Some("None".to_string()), + documentation_url: Some("None".to_string()), + created_at: Some("None".to_string()), + updated_at: Some("None".to_string()), + environment: Some("None".to_string()), + } + } +} + impl ServiceInfo { /// Get the id. pub fn id(&self) -> Option<&str> { @@ -421,7 +439,7 @@ impl ServiceInfo { impl Default for TicketServerConfig { fn default() -> Self { Self { - addr: default_addr().parse().expect("expected valid address"), + ticket_server_addr: default_addr().parse().expect("expected valid address"), cors: CorsConfig::default(), service_info: ServiceInfo::default(), } diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 8a8e9115f..83321e232 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -45,10 +45,11 @@ impl Default for StorageType { /// Schemes that can be used with htsget. #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "UPPERCASE")] pub enum Scheme { - #[serde(alias = "http", alias = "HTTP")] + #[serde(alias = "Http", alias = "http")] Http, - #[serde(alias = "https", alias = "HTTPS")] + #[serde(alias = "Https", alias = "https")] Https, } @@ -126,7 +127,7 @@ pub struct RegexResolver { // Todo: should match guard be allowed as variables inside the substitution string? substitution_string: String, storage_type: StorageType, - guard: QueryGuard, + allow_guard: QueryGuard, } with_prefix!(allow_interval_prefix "allow_interval_"); @@ -260,13 +261,13 @@ impl RegexResolver { storage_type: StorageType, regex: &str, replacement_string: &str, - guard: QueryGuard, + allow_guard: QueryGuard, ) -> Result { Ok(Self { regex: Regex::new(regex)?, substitution_string: replacement_string.to_string(), storage_type, - guard, + allow_guard, }) } @@ -281,8 +282,8 @@ impl RegexResolver { } /// Get the query guard. - pub fn guard(&self) -> &QueryGuard { - &self.guard + pub fn allow_guard(&self) -> &QueryGuard { + &self.allow_guard } /// Get the storage type. @@ -292,39 +293,39 @@ impl RegexResolver { /// Get allow formats. pub fn allow_formats(&self) -> &[Format] { - self.guard.allow_formats() + self.allow_guard.allow_formats() } /// Get allow classes. pub fn allow_classes(&self) -> &[Class] { - self.guard.allow_classes() + self.allow_guard.allow_classes() } /// Get allow interval. pub fn allow_interval(&self) -> Interval { - self.guard.allow_interval - } - - // /// Get allow reference names. - // pub fn allow_reference_names(&self) -> &ReferenceNames { - // &self.guard.allow_reference_names - // } - // - // /// Get allow fields. - // pub fn allow_fields(&self) -> &Fields { - // &self.guard.allow_fields - // } - // - // /// Get allow tags. - // pub fn allow_tags(&self) -> &Tags { - // &self.guard.allow_tags - // } + self.allow_guard.allow_interval + } + + /// Get allow reference names. + pub fn allow_reference_names(&self) -> &ReferenceNames { + &self.allow_guard.allow_reference_names + } + + /// Get allow fields. + pub fn allow_fields(&self) -> &Fields { + &self.allow_guard.allow_fields + } + + /// Get allow tags. + pub fn allow_tags(&self) -> &Tags { + &self.allow_guard.allow_tags + } } impl Resolver for RegexResolver { #[instrument(level = "trace", skip(self), ret)] fn resolve_id(&self, query: &Query) -> Option { - if self.regex.is_match(&query.id) && self.guard.query_matches(query) { + if self.regex.is_match(&query.id) && self.allow_guard.query_matches(query) { Some( self .regex From c4cc4d1ce316ca7cc179bde2807ff8fc36c742a1 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 22 Dec 2022 22:34:35 +1100 Subject: [PATCH 36/45] bug: fix broken data server optional by introducing boolean flag to enable data server --- htsget-config/README.md | 6 +- htsget-config/config.toml | 21 +++--- htsget-config/src/config/mod.rs | 75 ++++++------------- htsget-config/src/regex_resolver/mod.rs | 2 +- .../benches/request_benchmarks.rs | 2 +- htsget-http-actix/src/main.rs | 6 +- htsget-http-lambda/src/main.rs | 2 +- htsget-test-utils/src/http_tests.rs | 3 +- htsget-test-utils/src/server_tests.rs | 8 +- 9 files changed, 48 insertions(+), 77 deletions(-) diff --git a/htsget-config/README.md b/htsget-config/README.md index 65df42d2d..445507535 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -89,7 +89,7 @@ Sometimes it may be useful to disable the data server as all responses to the ti To disable the data server, set the following option: 
-data_server = ''
+data_server_enabled = false
#### Service info config @@ -228,7 +228,7 @@ the `--print-default-config` flag: cargo run -p htsget-http-actix -- --print-default-config ``` -Pass the `--help` flag to see more details on command line options: +Use the `--help` flag to see more details on command line options. [htsget-http-actix]: ../htsget-http-actix [htsget-http-lambda]: ../htsget-http-lambda @@ -298,7 +298,7 @@ Note the use of double quotes for certin values. Similar to the [data_server](#data_server) option, the data server can be disabled by setting the equivalent environment variable: ```shell -export HTSGET_DATA_SERVER="" +export HTSGET_DATA_SERVER_ENABLED=false ``` [service-info]: https://samtools.github.io/hts-specs/htsget.html#ga4gh-service-info diff --git a/htsget-config/config.toml b/htsget-config/config.toml index 67bb7ceb2..5f639e805 100644 --- a/htsget-config/config.toml +++ b/htsget-config/config.toml @@ -6,16 +6,17 @@ ticket_server_cors_allow_methods = 'All' ticket_server_cors_max_age = 86400 ticket_server_cors_expose_headers = [] -[data_server] -addr = '127.0.0.1:8081' -local_path = 'data' -serve_at = '/data' -cors_allow_credentials = false -cors_allow_origins = ['http://localhost:8080'] -cors_allow_headers = 'All' -cors_allow_methods = 'All' -cors_max_age = 86400 -cors_expose_headers = [] +data_server = "None" +#data_server_disabled = true +data_server_addr = '127.0.0.1:8082' +data_server_local_path = 'data' +data_server_serve_at = '/data' +data_server_cors_allow_credentials = false +data_server_cors_allow_origins = ['http://localhost:8080'] +data_server_cors_allow_headers = 'All' +data_server_cors_allow_methods = 'All' +data_server_cors_max_age = 86400 +data_server_cors_expose_headers = [] [[resolvers]] regex = '.*' diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 0bea686ce..fda5dabe0 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -104,23 +104,10 @@ pub struct Config { #[serde(flatten)] ticket_server: TicketServerConfig, #[serde(flatten, with = "data_server_prefix")] - data_server: DataServerConfigOption, + data_server: DataServerConfig, resolvers: Vec, } -#[derive(Serialize, Deserialize, Debug, Clone)] -enum DataServerConfigNone { - #[serde(alias = "none", alias = "NONE", alias = "")] - None, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(untagged)] -enum DataServerConfigOption { - None(DataServerConfigNone), - Some(DataServerConfig), -} - with_prefix!(ticket_server_cors_prefix "ticket_server_cors_"); /// Configuration for the htsget ticket server. @@ -246,6 +233,7 @@ with_prefix!(cors_prefix "cors_"); #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct DataServerConfig { + enabled: bool, addr: SocketAddr, local_path: PathBuf, serve_at: PathBuf, @@ -258,6 +246,7 @@ pub struct DataServerConfig { impl DataServerConfig { /// Create a new data server config. pub fn new( + enabled: bool, addr: SocketAddr, local_path: PathBuf, serve_at: PathBuf, @@ -266,6 +255,7 @@ impl DataServerConfig { cors: CorsConfig, ) -> Self { Self { + enabled, addr, local_path, serve_at, @@ -334,11 +324,17 @@ impl DataServerConfig { pub fn expose_headers(&self) -> &AllowType { self.cors.expose_headers() } + + /// Is the data server disabled + pub fn enabled(&self) -> bool { + self.enabled + } } impl Default for DataServerConfig { fn default() -> Self { Self { + enabled: true, addr: default_localstorage_addr() .parse() .expect("expected valid address"), @@ -352,7 +348,7 @@ impl Default for DataServerConfig { } /// Configuration of the service info. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct ServiceInfo { id: Option, @@ -367,23 +363,6 @@ pub struct ServiceInfo { environment: Option, } -impl Default for ServiceInfo { - fn default() -> Self { - Self { - id: Some("None".to_string()), - name: Some("None".to_string()), - version: Some("None".to_string()), - organization_name: Some("None".to_string()), - organization_url: Some("None".to_string()), - contact_url: Some("None".to_string()), - documentation_url: Some("None".to_string()), - created_at: Some("None".to_string()), - updated_at: Some("None".to_string()), - environment: Some("None".to_string()), - } - } -} - impl ServiceInfo { /// Get the id. pub fn id(&self) -> Option<&str> { @@ -450,7 +429,7 @@ impl Default for Config { fn default() -> Self { Self { ticket_server: TicketServerConfig::default(), - data_server: DataServerConfigOption::Some(DataServerConfig::default()), + data_server: DataServerConfig::default(), resolvers: vec![RegexResolver::default()], } } @@ -460,15 +439,12 @@ impl Config { /// Create a new config. pub fn new( ticket_server: TicketServerConfig, - data_server: Option, + data_server: DataServerConfig, resolvers: Vec, ) -> Self { Self { ticket_server, - data_server: match data_server { - None => DataServerConfigOption::None(DataServerConfigNone::None), - Some(value) => DataServerConfigOption::Some(value), - }, + data_server, resolvers, } } @@ -490,7 +466,7 @@ impl Config { /// Read the environment variables into a Config struct. #[instrument] - pub fn from_env(config: PathBuf) -> io::Result { + pub fn from_config(config: PathBuf) -> io::Result { let config = Figment::from(Serialized::defaults(Config::default())) .merge(Toml::file(config)) .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX)) @@ -526,11 +502,8 @@ impl Config { } /// Get the data server. - pub fn data_server(&self) -> Option<&DataServerConfig> { - match self.data_server { - DataServerConfigOption::None(_) => None, - DataServerConfigOption::Some(ref config) => Some(config), - } + pub fn data_server(&self) -> &DataServerConfig { + &self.data_server } /// Get the resolvers. @@ -567,7 +540,7 @@ mod tests { jail.set_env(key, value); } - test_fn(Config::from_env("test.toml".into()).map_err(|err| err.to_string())?); + test_fn(Config::from_config("test.toml".into()).map_err(|err| err.to_string())?); Ok(()) }); @@ -625,7 +598,7 @@ mod tests { vec![("HTSGET_DATA_SERVER", "{addr=127.0.0.1:8082}")], |config| { assert_eq!( - config.data_server().unwrap().addr(), + config.data_server().addr(), "127.0.0.1:8082".parse().unwrap() ); }, @@ -634,8 +607,8 @@ mod tests { #[test] fn config_no_data_server_env() { - test_config_from_env(vec![("HTSGET_DATA_SERVER", "")], |config| { - assert!(matches!(config.data_server(), None)); + test_config_from_env(vec![("HTSGET_DATA_SERVER_ENABLED", "false")], |config| { + assert!(config.data_server().enabled()); }); } @@ -682,7 +655,7 @@ mod tests { "#, |config| { assert_eq!( - config.data_server().unwrap().addr(), + config.data_server().addr(), "127.0.0.1:8082".parse().unwrap() ); }, @@ -691,8 +664,8 @@ mod tests { #[test] fn config_no_data_server_file() { - test_config_from_file(r#"data_server = """#, |config| { - assert!(matches!(config.data_server(), None)); + test_config_from_file(r#"data_server_enabled = false"#, |config| { + assert!(config.data_server().enabled()); }); } diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 83321e232..a3a916236 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -30,7 +30,7 @@ pub trait QueryMatcher { #[serde(tag = "type")] #[non_exhaustive] pub enum StorageType { - #[serde(alias = "url", alias = "URL")] + #[serde(alias = "local", alias = "LOCAL")] Local(LocalResolver), #[cfg(feature = "s3-storage")] #[serde(alias = "s3")] diff --git a/htsget-http-actix/benches/request_benchmarks.rs b/htsget-http-actix/benches/request_benchmarks.rs index c9075dbd1..91864609c 100644 --- a/htsget-http-actix/benches/request_benchmarks.rs +++ b/htsget-http-actix/benches/request_benchmarks.rs @@ -151,7 +151,7 @@ fn start_htsget_rs() -> (DropGuard, String) { let htsget_rs_url = format!("http://{}", config.ticket_server().addr()); query_server_until_response(&format_url(&htsget_rs_url, "reads/service-info")); - let htsget_rs_ticket_url = format!("http://{}", config.data_server().unwrap().addr()); + let htsget_rs_ticket_url = format!("http://{}", config.data_server().addr()); query_server_until_response(&format_url(&htsget_rs_ticket_url, "")); (DropGuard(child), htsget_rs_url) diff --git a/htsget-http-actix/src/main.rs b/htsget-http-actix/src/main.rs index 1d8912688..d50af2c9b 100644 --- a/htsget-http-actix/src/main.rs +++ b/htsget-http-actix/src/main.rs @@ -9,10 +9,10 @@ async fn main() -> std::io::Result<()> { Config::setup_tracing()?; if let Some(config) = Config::parse_args() { - let config = Config::from_env(config)?; + let config = Config::from_config(config)?; - if let Some(server) = config.data_server() { - let server = server.clone(); + if config.data_server().enabled() { + let server = config.data_server().clone(); let mut formatter = HttpTicketFormatter::try_from(server.clone())?; let local_server = formatter.bind_data_server().await?; diff --git a/htsget-http-lambda/src/main.rs b/htsget-http-lambda/src/main.rs index b954648ed..9c1afa973 100644 --- a/htsget-http-lambda/src/main.rs +++ b/htsget-http-lambda/src/main.rs @@ -9,7 +9,7 @@ async fn main() -> Result<(), Error> { Config::setup_tracing()?; if let Some(config) = Config::parse_args() { - let config = Config::from_env(config)?; + let config = Config::from_config(config)?; let service_info = config.ticket_server().service_info().clone(); let cors = config.ticket_server().cors().clone(); diff --git a/htsget-test-utils/src/http_tests.rs b/htsget-test-utils/src/http_tests.rs index a3651ab18..ff8d44a6f 100644 --- a/htsget-test-utils/src/http_tests.rs +++ b/htsget-test-utils/src/http_tests.rs @@ -134,6 +134,7 @@ fn default_test_config_params( ) -> Config { let cors = default_cors_config(); let server_config = DataServerConfig::new( + true, addr, default_dir_data(), PathBuf::from("/data"), @@ -144,7 +145,7 @@ fn default_test_config_params( Config::new( TicketServerConfig::new("127.0.0.1:8080".parse().unwrap(), cors, Default::default()), - Some(server_config), + server_config, vec![default_test_resolver(addr, scheme)], ) } diff --git a/htsget-test-utils/src/server_tests.rs b/htsget-test-utils/src/server_tests.rs index 19668cf0c..a17546205 100644 --- a/htsget-test-utils/src/server_tests.rs +++ b/htsget-test-utils/src/server_tests.rs @@ -77,11 +77,7 @@ pub async fn test_response(response: Response, class: Class) { /// Create the a [HttpTicketFormatter], spawn the ticket server, returning the expected path and the formatter. pub async fn formatter_and_expected_path(config: &Config) -> (String, HttpTicketFormatter) { let mut formatter = formatter_from_config(config).unwrap(); - spawn_ticket_server( - config.data_server().unwrap().local_path().into(), - &mut formatter, - ) - .await; + spawn_ticket_server(config.data_server().local_path().into(), &mut formatter).await; (expected_url_path(&formatter), formatter) } @@ -170,7 +166,7 @@ pub async fn test_parameterized_post_class_header(tester: &impl /// Get the [HttpTicketFormatter] from the config. pub fn formatter_from_config(config: &Config) -> Option { - HttpTicketFormatter::try_from(config.data_server().unwrap().clone()).ok() + HttpTicketFormatter::try_from(config.data_server().clone()).ok() } /// A service info test. From 508763ec5df8abfa826d5b39bb133bd507586c1c Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 23 Dec 2022 08:29:51 +1100 Subject: [PATCH 37/45] test: add test for long resolvers from environment variable config --- htsget-config/README.md | 26 +++++---- htsget-config/src/config/mod.rs | 66 +++++++++++++++++------ htsget-config/src/regex_resolver/aws.rs | 7 ++- htsget-config/src/regex_resolver/mod.rs | 71 ++++++++++++++++--------- htsget-search/src/lib.rs | 2 +- 5 files changed, 113 insertions(+), 59 deletions(-) diff --git a/htsget-config/README.md b/htsget-config/README.md index 445507535..3bf41badd 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -166,9 +166,9 @@ If the type is `Local`, then the following options can be set: If the type is `S3`, then the following option can be set: -| Option | Description | Type | Default | -|----------|----------------------------------------------------------|-----------------------------|----------| -| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | Not set | +| Option | Description | Type | Default | +|----------|----------------------------------------------------------|-----------------------------|-------| +| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | `''` | Additionally, the resolver component has a feature, which allows resolving IDs based on the other fields present in a query. This is useful as allows the resolver to match only match an ID, if a particular set of query parameters are also present. For example, @@ -275,26 +275,24 @@ In order to use `HTSGET_RESOLVERS`, the entire resolver config array must be set ```shell export HTSGET_RESOLVERS="[{ - regex="regex", - substitution_string="substitution_string", + regex=regex, + substitution_string=substitution_string, storage_type={ - type="S3", - bucket="bucket" + type=S3, + bucket=bucket }, allow_guard={ - allow_reference_names="['chr1']", - allow_fields="['QNAME']", - allow_tags="['RG']", - allow_formats="['BAM']", - allow_classes="['body']", + allow_reference_names=[chr1], + allow_fields=[QNAME], + allow_tags=[RG], + allow_formats=[BAM], + allow_classes=[body], allow_interval_start=100, allow_interval_end=1000 } }]" ``` -Note the use of double quotes for certin values. - Similar to the [data_server](#data_server) option, the data server can be disabled by setting the equivalent environment variable: ```shell diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index fda5dabe0..68045a855 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -520,9 +520,12 @@ impl Config { #[cfg(test)] mod tests { use super::*; - use crate::regex_resolver::{Scheme, StorageType}; + use crate::regex_resolver::aws::S3Resolver; + use crate::regex_resolver::{AllowGuard, ReferenceNames, Scheme, StorageType}; use crate::Format::Bam; + use crate::{Class, Fields, Interval, Tags}; use figment::Jail; + use std::collections::HashSet; use std::fmt::Display; fn test_config(contents: Option<&str>, env_variables: Vec<(K, V)>, test_fn: F) @@ -595,7 +598,7 @@ mod tests { #[test] fn config_data_server_addr_env() { test_config_from_env( - vec![("HTSGET_DATA_SERVER", "{addr=127.0.0.1:8082}")], + vec![("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082")], |config| { assert_eq!( config.data_server().addr(), @@ -607,7 +610,7 @@ mod tests { #[test] fn config_no_data_server_env() { - test_config_from_env(vec![("HTSGET_DATA_SERVER_ENABLED", "false")], |config| { + test_config_from_env(vec![("HTSGET_DATA_SERVER_ENABLED", "true")], |config| { assert!(config.data_server().enabled()); }); } @@ -622,6 +625,40 @@ mod tests { }); } + #[test] + fn config_resolvers_all_options_env() { + test_config_from_env( + vec![( + "HTSGET_RESOLVERS", + "[{ regex=regex, substitution_string=substitution_string, \ + storage_type={ type=S3, bucket=bucket }, \ + allow_guard={ allow_reference_names=[chr1], allow_fields=[QNAME], allow_tags=[RG], \ + allow_formats=[BAM], allow_classes=[body], allow_interval_start=100, \ + allow_interval_end=1000 } }]", + )], + |config| { + let storage_type = StorageType::S3(S3Resolver::new("bucket".to_string())); + let allow_guard = AllowGuard::new( + ReferenceNames::List(HashSet::from_iter(vec!["chr1".to_string()])), + Fields::List(HashSet::from_iter(vec!["QNAME".to_string()])), + Tags::List(HashSet::from_iter(vec!["RG".to_string()])), + vec![Bam], + vec![Class::Body], + Interval { + start: Some(100), + end: Some(1000), + }, + ); + let resolver = config.resolvers.first().unwrap(); + + assert_eq!(resolver.regex().to_string(), "regex"); + assert_eq!(resolver.substitution_string(), "substitution_string"); + assert_eq!(resolver.storage_type(), &storage_type); + assert_eq!(resolver.allow_guard(), &allow_guard); + }, + ); + } + #[test] fn config_ticket_server_addr_file() { test_config_from_file(r#"ticket_server_addr = "127.0.0.1:8082""#, |config| { @@ -648,23 +685,17 @@ mod tests { #[test] fn config_data_server_addr_file() { - test_config_from_file( - r#" - [data_server] - addr = "127.0.0.1:8082" - "#, - |config| { - assert_eq!( - config.data_server().addr(), - "127.0.0.1:8082".parse().unwrap() - ); - }, - ); + test_config_from_file(r#"data_server_addr = "127.0.0.1:8082""#, |config| { + assert_eq!( + config.data_server().addr(), + "127.0.0.1:8082".parse().unwrap() + ); + }); } #[test] fn config_no_data_server_file() { - test_config_from_file(r#"data_server_enabled = false"#, |config| { + test_config_from_file(r#"data_server_enabled = true"#, |config| { assert!(config.data_server().enabled()); }); } @@ -692,7 +723,7 @@ mod tests { [[resolvers]] regex = "regex" - [resolvers.guard] + [resolvers.allow_guard] allow_formats = ["BAM"] "#, |config| { @@ -718,6 +749,7 @@ mod tests { path_prefix = "path" "#, |config| { + println!("{:?}", config.resolvers().first().unwrap().storage_type()); assert!(matches!( config.resolvers().first().unwrap().storage_type(), StorageType::Local(resolver) if resolver.local_path() == "path" && resolver.scheme() == Scheme::Https && resolver.path_prefix() == "path" diff --git a/htsget-config/src/regex_resolver/aws.rs b/htsget-config/src/regex_resolver/aws.rs index b81ce6e7b..ab9d1b976 100644 --- a/htsget-config/src/regex_resolver/aws.rs +++ b/htsget-config/src/regex_resolver/aws.rs @@ -2,13 +2,18 @@ use serde; use serde::{Deserialize, Serialize}; /// S3 configuration for the htsget server. -#[derive(Deserialize, Serialize, Debug, Clone, Default)] +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq, Eq)] #[serde(default)] pub struct S3Resolver { bucket: String, } impl S3Resolver { + /// Create a new S3 resolver. + pub fn new(bucket: String) -> Self { + Self { bucket } + } + /// Get the bucket. pub fn bucket(&self) -> &str { &self.bucket diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index a3a916236..5826a1c87 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -20,13 +20,13 @@ pub trait Resolver { } /// Determines whether the query matches for use with the resolver. -pub trait QueryMatcher { +pub trait QueryAllowed { /// Does this query match. - fn query_matches(&self, query: &Query) -> bool; + fn query_allowed(&self, query: &Query) -> bool; } /// Specify the storage type to use. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(tag = "type")] #[non_exhaustive] pub enum StorageType { @@ -60,7 +60,7 @@ impl Default for Scheme { } /// A local resolver, which can return files from the local file system. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(default)] pub struct LocalResolver { scheme: Scheme, @@ -127,15 +127,15 @@ pub struct RegexResolver { // Todo: should match guard be allowed as variables inside the substitution string? substitution_string: String, storage_type: StorageType, - allow_guard: QueryGuard, + allow_guard: AllowGuard, } with_prefix!(allow_interval_prefix "allow_interval_"); /// A query guard represents query parameters that can be allowed to resolver for a given query. -#[derive(Serialize, Clone, Debug, Deserialize)] +#[derive(Serialize, Clone, Debug, Deserialize, PartialEq, Eq)] #[serde(default)] -pub struct QueryGuard { +pub struct AllowGuard { allow_reference_names: ReferenceNames, allow_fields: Fields, allow_tags: Tags, @@ -146,14 +146,33 @@ pub struct QueryGuard { } /// Reference names that can be matched. -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(untagged)] pub enum ReferenceNames { Tagged(TaggedTypeAll), List(HashSet), } -impl QueryGuard { +impl AllowGuard { + /// Create a new allow guard. + pub fn new( + allow_reference_names: ReferenceNames, + allow_fields: Fields, + allow_tags: Tags, + allow_formats: Vec, + allow_classes: Vec, + allow_interval: Interval, + ) -> Self { + Self { + allow_reference_names, + allow_fields, + allow_tags, + allow_formats, + allow_classes, + allow_interval, + } + } + /// Get allow formats. pub fn allow_formats(&self) -> &[Format] { &self.allow_formats @@ -185,7 +204,7 @@ impl QueryGuard { } } -impl Default for QueryGuard { +impl Default for AllowGuard { fn default() -> Self { Self { allow_formats: vec![Bam, Cram, Vcf, Bcf], @@ -198,8 +217,8 @@ impl Default for QueryGuard { } } -impl QueryMatcher for ReferenceNames { - fn query_matches(&self, query: &Query) -> bool { +impl QueryAllowed for ReferenceNames { + fn query_allowed(&self, query: &Query) -> bool { match (self, &query.reference_name) { (ReferenceNames::Tagged(TaggedTypeAll::All), _) => true, (ReferenceNames::List(reference_names), Some(reference_name)) => { @@ -210,8 +229,8 @@ impl QueryMatcher for ReferenceNames { } } -impl QueryMatcher for Fields { - fn query_matches(&self, query: &Query) -> bool { +impl QueryAllowed for Fields { + fn query_allowed(&self, query: &Query) -> bool { match (self, &query.fields) { (Fields::Tagged(TaggedTypeAll::All), _) => true, (Fields::List(self_fields), Fields::List(query_fields)) => { @@ -222,8 +241,8 @@ impl QueryMatcher for Fields { } } -impl QueryMatcher for Tags { - fn query_matches(&self, query: &Query) -> bool { +impl QueryAllowed for Tags { + fn query_allowed(&self, query: &Query) -> bool { match (self, &query.tags) { (Tags::Tagged(TaggedTypeAll::All), _) => true, (Tags::List(self_tags), Tags::List(query_tags)) => self_tags.is_subset(query_tags), @@ -232,8 +251,8 @@ impl QueryMatcher for Tags { } } -impl QueryMatcher for QueryGuard { - fn query_matches(&self, query: &Query) -> bool { +impl QueryAllowed for AllowGuard { + fn query_allowed(&self, query: &Query) -> bool { self.allow_formats.contains(&query.format) && self.allow_classes.contains(&query.class) && self @@ -242,15 +261,15 @@ impl QueryMatcher for QueryGuard { && self .allow_interval .contains(query.interval.end.unwrap_or(u32::MAX)) - && self.allow_reference_names.query_matches(query) - && self.allow_fields.query_matches(query) - && self.allow_tags.query_matches(query) + && self.allow_reference_names.query_allowed(query) + && self.allow_fields.query_allowed(query) + && self.allow_tags.query_allowed(query) } } impl Default for RegexResolver { fn default() -> Self { - Self::new(StorageType::default(), ".*", "$0", QueryGuard::default()) + Self::new(StorageType::default(), ".*", "$0", AllowGuard::default()) .expect("expected valid resolver") } } @@ -261,7 +280,7 @@ impl RegexResolver { storage_type: StorageType, regex: &str, replacement_string: &str, - allow_guard: QueryGuard, + allow_guard: AllowGuard, ) -> Result { Ok(Self { regex: Regex::new(regex)?, @@ -282,7 +301,7 @@ impl RegexResolver { } /// Get the query guard. - pub fn allow_guard(&self) -> &QueryGuard { + pub fn allow_guard(&self) -> &AllowGuard { &self.allow_guard } @@ -325,7 +344,7 @@ impl RegexResolver { impl Resolver for RegexResolver { #[instrument(level = "trace", skip(self), ret)] fn resolve_id(&self, query: &Query) -> Option { - if self.regex.is_match(&query.id) && self.allow_guard.query_matches(query) { + if self.regex.is_match(&query.id) && self.allow_guard.query_allowed(query) { Some( self .regex @@ -348,7 +367,7 @@ pub mod tests { StorageType::default(), ".*", "$0-test", - QueryGuard::default(), + AllowGuard::default(), ) .unwrap(); assert_eq!( diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index e3d16426b..ab02bd596 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -2,7 +2,7 @@ pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketSer #[cfg(feature = "s3-storage")] pub use htsget_config::regex_resolver::aws::S3Resolver; pub use htsget_config::regex_resolver::{ - LocalResolver, QueryMatcher, RegexResolver, Resolver, StorageType, + LocalResolver, QueryAllowed, RegexResolver, Resolver, StorageType, }; pub mod htsget; From 5ab9f96468d63c7e8f2517833c71f5921f84c969 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 23 Dec 2022 08:33:49 +1100 Subject: [PATCH 38/45] build: fix feature flag compile errors --- htsget-config/src/config/mod.rs | 8 +++++++- htsget-config/src/regex_resolver/mod.rs | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 68045a855..ea09fecbf 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -520,11 +520,16 @@ impl Config { #[cfg(test)] mod tests { use super::*; + #[cfg(feature = "s3-storage")] use crate::regex_resolver::aws::S3Resolver; - use crate::regex_resolver::{AllowGuard, ReferenceNames, Scheme, StorageType}; + #[cfg(feature = "s3-storage")] + use crate::regex_resolver::{AllowGuard, ReferenceNames}; + use crate::regex_resolver::{Scheme, StorageType}; use crate::Format::Bam; + #[cfg(feature = "s3-storage")] use crate::{Class, Fields, Interval, Tags}; use figment::Jail; + #[cfg(feature = "s3-storage")] use std::collections::HashSet; use std::fmt::Display; @@ -625,6 +630,7 @@ mod tests { }); } + #[cfg(feature = "s3-storage")] #[test] fn config_resolvers_all_options_env() { test_config_from_env( diff --git a/htsget-config/src/regex_resolver/mod.rs b/htsget-config/src/regex_resolver/mod.rs index 5826a1c87..708873b36 100644 --- a/htsget-config/src/regex_resolver/mod.rs +++ b/htsget-config/src/regex_resolver/mod.rs @@ -6,6 +6,7 @@ use std::collections::HashSet; use tracing::instrument; use crate::config::{default_localstorage_addr, default_path, default_serve_at}; +#[cfg(feature = "s3-storage")] use crate::regex_resolver::aws::S3Resolver; use crate::Format::{Bam, Bcf, Cram, Vcf}; use crate::{Class, Fields, Format, Interval, Query, TaggedTypeAll, Tags}; From 963f6e8dbcaace1e0cfd234795ddf0e820dc8850 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 23 Dec 2022 11:51:17 +1100 Subject: [PATCH 39/45] bug: remove duplicate config module --- htsget-config/src/config.rs | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 htsget-config/src/config.rs diff --git a/htsget-config/src/config.rs b/htsget-config/src/config.rs deleted file mode 100644 index e69de29bb..000000000 From fde1f8602d69a247587d3ee41c5ceb56db47748a Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 23 Dec 2022 11:51:27 +1100 Subject: [PATCH 40/45] docs: reword usage string --- htsget-config/src/config/mod.rs | 38 +++------------------------------ 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index ea09fecbf..b042d9f0d 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -22,41 +22,9 @@ use tracing_subscriber::{fmt, EnvFilter, Registry}; use crate::regex_resolver::RegexResolver; /// Represents a usage string for htsget-rs. -pub const USAGE: &str = r#" -Available environment variables: -* HTSGET_PATH: The path to the directory where the server should be started. Default: "data". Unused if HTSGET_STORAGE_TYPE is "AwsS3Storage". -* HTSGET_REGEX: The regular expression that should match an ID. Default: ".*". -For more information about the regex options look in the documentation of the regex crate(https://docs.rs/regex/). -* HTSGET_SUBSTITUTION_STRING: The replacement expression. Default: "$0". -* HTSGET_STORAGE_TYPE: Either "LocalStorage" or "AwsS3Storage", representing which storage type to use. Default: "LocalStorage". - -The following options are used for the ticket server. -* HTSGET_TICKET_SERVER_ADDR: The socket address for the server which creates response tickets. Default: "127.0.0.1:8080". -* HTSGET_TICKET_SERVER_ALLOW_CREDENTIALS: Boolean flag, indicating whether authenticated requests are allowed by including the `Access-Control-Allow-Credentials` header. Default: "false". -* HTSGET_TICKET_SERVER_ALLOW_ORIGIN: Which origin os allowed in the `ORIGIN` header. Default: "http://localhost:8080". - -The following options are used for the data server. -* HTSGET_DATA_SERVER_ADDR: The socket address to use for the server which responds to tickets. Default: "127.0.0.1:8081". Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". -* HTSGET_DATA_SERVER_KEY: The path to the PEM formatted X.509 private key used by the data server. Default: "None". Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". -* HTSGET_DATA_SERVER_CERT: The path to the PEM formatted X.509 certificate used by the data server. Default: "None". Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". -* HTSGET_DATA_SERVER_ALLOW_CREDENTIALS: Boolean flag, indicating whether authenticated requests are allowed by including the `Access-Control-Allow-Credentials` header. Default: "false" -* HTSGET_DATA_SERVER_ALLOW_ORIGIN: Which origin os allowed in the `ORIGIN` header. Default: "http://localhost:8081" - -The following options are used to configure AWS S3 storage. -* HTSGET_S3_BUCKET: The name of the AWS S3 bucket. Default: "". Unused if HTSGET_STORAGE_TYPE is not "AwsS3Storage". - -The next variables are used to configure the info for the service-info endpoints. -* HTSGET_ID: The id of the service. Default: "None". -* HTSGET_NAME: The name of the service. Default: "None". -* HTSGET_VERSION: The version of the service. Default: "None". -* HTSGET_ORGANIZATION_NAME: The name of the organization. Default: "None". -* HTSGET_ORGANIZATION_URL: The url of the organization. Default: "None". -* HTSGET_CONTACT_URL: A url to provide contact to the users. Default: "None". -* HTSGET_DOCUMENTATION_URL: A link to the documentation. Default: "None". -* HTSGET_CREATED_AT: Date of the creation of the service. Default: "None". -* HTSGET_UPDATED_AT: Date of the last update of the service. Default: "None". -* HTSGET_ENVIRONMENT: The environment in which the service is running. Default: "None". -"#; +pub const USAGE: &str = + "htsget-rs can be configured using a config file or environment variables. \ +See the documentation of the htsget-config crate for more information."; const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET_"; From 10d86bf6157e29f6932d66ded9d38129c89cb87c Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 23 Dec 2022 15:04:31 +1100 Subject: [PATCH 41/45] docs: clarify how the resolvers work --- htsget-config/README.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/htsget-config/README.md b/htsget-config/README.md index 3bf41badd..3c06e84b8 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -166,15 +166,15 @@ If the type is `Local`, then the following options can be set: If the type is `S3`, then the following option can be set: -| Option | Description | Type | Default | -|----------|----------------------------------------------------------|-----------------------------|-------| -| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | `''` | +| Option | Description | Type | Default | +|----------|----------------------------------------------------------|-----------------|---------| +| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | `''` | Additionally, the resolver component has a feature, which allows resolving IDs based on the other fields present in a query. -This is useful as allows the resolver to match only match an ID, if a particular set of query parameters are also present. For example, +This is useful as allows the resolver to match an ID, if a particular set of query parameters are also present. For example, a resolver can be set to only resolve IDs if the format is also BAM. -To set the resolver 'allow guard', add a `[resolver.allow_guard]` table, and set the following options: +This component can be configured by setting the `[resolver.allow_guard]` table with. The following options are available to restrict which queries are resolved by a resolver: | Option | Description | Type | Default | |-------------------------|-----------------------------------------------------------------------------------------|-----------------------------------------------------------------------|-------------------------------------| @@ -207,6 +207,8 @@ allow_interval_start = 100 allow_interval_end = 1000 ``` +In this example, the resolver will only match the query ID if the query is for `chr1` with positions between `100` and `1000`. + #### Config file location The htsget-rs binaries ([htsget-http-actix] and [htsget-http-lambda]) support some command line options. The config file location can @@ -235,7 +237,8 @@ Use the `--help` flag to see more details on command line options. #### Configuring htsget-rs with environment variables -All the htsget-rs config options can be set by environment variables. The ticket server, data server and service info options are flattened and can be set directly using +All the htsget-rs config options can be set by environment variables, which is convenient for runtimes such as AWS Lambda. +The ticket server, data server and service info options are flattened and can be set directly using environment variable. It is not recommended to set the resolvers using environment variables, however it can be done by setting a single environment variable which contains a list of structures, where a key name and value pair is used to set the nested options. From 93be5a22e246d492de698a8f9afa46ab80b8ba80 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 23 Dec 2022 15:06:42 +1100 Subject: [PATCH 42/45] refactor: remove some unnecessary unwraps --- htsget-http-lambda/src/lib.rs | 20 ++++++++++---------- htsget-test-utils/src/server_tests.rs | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/htsget-http-lambda/src/lib.rs b/htsget-http-lambda/src/lib.rs index 826abc8a9..ca35764d7 100644 --- a/htsget-http-lambda/src/lib.rs +++ b/htsget-http-lambda/src/lib.rs @@ -485,7 +485,7 @@ mod tests { assert!(router.get_route(&Method::DELETE, &uri).is_none()); }, &config, - formatter_from_config(&config).unwrap(), + formatter_from_config(&config), ) .await; } @@ -499,7 +499,7 @@ mod tests { assert!(router.get_route(&Method::GET, &uri).is_none()); }, &config, - formatter_from_config(&config).unwrap(), + formatter_from_config(&config), ) .await; } @@ -513,7 +513,7 @@ mod tests { assert!(router.get_route(&Method::GET, &uri).is_none()); }, &config, - formatter_from_config(&config).unwrap(), + formatter_from_config(&config), ) .await; } @@ -527,7 +527,7 @@ mod tests { assert!(router.get_route(&Method::GET, &uri).is_none()); }, &config, - formatter_from_config(&config).unwrap(), + formatter_from_config(&config), ) .await; } @@ -541,7 +541,7 @@ mod tests { assert!(router.get_route(&Method::GET, &uri).is_none()); }, &config, - formatter_from_config(&config).unwrap(), + formatter_from_config(&config), ) .await; } @@ -566,7 +566,7 @@ mod tests { ); }, &config, - formatter_from_config(&config).unwrap(), + formatter_from_config(&config), ) .await; } @@ -591,7 +591,7 @@ mod tests { ); }, &config, - formatter_from_config(&config).unwrap(), + formatter_from_config(&config), ) .await; } @@ -613,7 +613,7 @@ mod tests { ); }, &config, - formatter_from_config(&config).unwrap(), + formatter_from_config(&config), ) .await; } @@ -638,7 +638,7 @@ mod tests { ); }, &config, - formatter_from_config(&config).unwrap(), + formatter_from_config(&config), ) .await; } @@ -680,7 +680,7 @@ mod tests { } async fn test_service_info_from_file(file_path: &str, config: &Config) { - let formatter = formatter_from_config(config).unwrap(); + let formatter = formatter_from_config(config); let expected_path = expected_url_path(&formatter); with_router( |router| async { diff --git a/htsget-test-utils/src/server_tests.rs b/htsget-test-utils/src/server_tests.rs index a17546205..acdb9f9d0 100644 --- a/htsget-test-utils/src/server_tests.rs +++ b/htsget-test-utils/src/server_tests.rs @@ -76,7 +76,7 @@ pub async fn test_response(response: Response, class: Class) { /// Create the a [HttpTicketFormatter], spawn the ticket server, returning the expected path and the formatter. pub async fn formatter_and_expected_path(config: &Config) -> (String, HttpTicketFormatter) { - let mut formatter = formatter_from_config(config).unwrap(); + let mut formatter = formatter_from_config(config); spawn_ticket_server(config.data_server().local_path().into(), &mut formatter).await; (expected_url_path(&formatter), formatter) @@ -165,8 +165,8 @@ pub async fn test_parameterized_post_class_header(tester: &impl } /// Get the [HttpTicketFormatter] from the config. -pub fn formatter_from_config(config: &Config) -> Option { - HttpTicketFormatter::try_from(config.data_server().clone()).ok() +pub fn formatter_from_config(config: &Config) -> HttpTicketFormatter { + HttpTicketFormatter::try_from(config.data_server().clone()).unwrap() } /// A service info test. From 512c1f0e86c4f6197d10596109bf9f0bd170f91c Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 23 Dec 2022 15:10:29 +1100 Subject: [PATCH 43/45] docs: reword resolvers description --- htsget-config/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/htsget-config/README.md b/htsget-config/README.md index 3c06e84b8..d00d09cb5 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -128,8 +128,7 @@ environment = 'dev' The resolvers component of htsget-rs is used to map query IDs to the location of the resource. Each query that htsget-rs receives is 'resolved' to a location, which a data server can respond with. A query ID is matched with a regex, and is then mapped with a substitution string that -has access to the regex capture groups. Each resolver is an array of TOML of tables that attempts to match a query ID. This array matches IDs in order, meaning that -the first matching resolver is resolver used to map the ID. +has access to the regex capture groups. Resolvers are configured in an array, where the first matching resolver is resolver used to map the ID. To create a resolver, add a `[[resolvers]]` array of tables, and set the following options: From 95ed6153e9cee29c9236bb54ec769253b6507b9b Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 23 Dec 2022 15:22:08 +1100 Subject: [PATCH 44/45] docs: remove unnecessary default column for environment variables, surround environment variables in backticks. --- htsget-config/README.md | 66 ++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/htsget-config/README.md b/htsget-config/README.md index d00d09cb5..f48fb0e12 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -156,12 +156,12 @@ To set the storage type for a resolver, add a `[resolvers.storage_type]` table. If the type is `Local`, then the following options can be set: -| Option | Description | Type | Default | -|---------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|---------------------| -| `scheme` | The scheme present on URL tickets. | Either `'HTTP'` or `'HTTPS'` | `'HTTP'` | -| `authority` | The authority present on URL tickets. This should likely match the `data_server_addr`. | URL authority | `'127.0.0.1:8081' ` | -| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server_local_path`. | Filesystem path | `'data'` | -| `path_prefix` | The path prefix which the URL tickets will have. This should likely match the `data_server_serve_at` path. | URL path | `'/data'` | +| Option | Description | Type | Default | +|---------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------| +| `scheme` | The scheme present on URL tickets. | Either `'HTTP'` or `'HTTPS'` | `'HTTP'` | +| `authority` | The authority present on URL tickets. This should likely match the `data_server_addr`. | URL authority | `'127.0.0.1:8081'` | +| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server_local_path`. | Filesystem path | `'data'` | +| `path_prefix` | The path prefix which the URL tickets will have. This should likely match the `data_server_serve_at` path. | URL path | `'/data'` | If the type is `S3`, then the following option can be set: @@ -245,33 +245,33 @@ Environment variables will override options set in the config file. Note, arrays The following environment variables - corresponding to the TOML config - are available: -| Variable | Description | Default | -|---------------------------------------------|-------------------------------------------------------------------------------------|-------------------------| -| HTSGET_TICKET_SERVER_ADDR | See [`ticket_server_addr`](#ticket_server_addr) | "data" | -| HTSGET_TICKET_SERVER_CORS_ALLOW_CREDENTIALS | See [`ticket_server_cors_allow_credentials`](#ticket_server_cors_allow_credentials) | ".*" | -| HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGINS | See [`ticket_server_cors_allow_origins`](#ticket_server_cors_allow_origins) | "$0" | -| HTSGET_TICKET_SERVER_CORS_ALLOW_HEADERS | See [`ticket_server_cors_allow_headers`](#ticket_server_cors_allow_headers) | "LocalStorage" | -| HTSGET_TICKET_SERVER_CORS_MAX_AGE | See [`ticket_server_cors_max_age`](#ticket_server_cors_max_age) | "127.0.0.1:8080" | -| HTSGET_TICKET_SERVER_CORS_EXPOSE_HEADERS | See [`ticket_server_cors_expose_headers`](#ticket_server_cors_expose_headers) | "false" | -| HTSGET_DATA_SERVER_ADDR | See [`data_server_addr`](#data_server_addr) | "127.0.0.1:8081" | -| HTSGET_DATA_SERVER_LOCAL_PATH | See [`data_server_local_path`](#data_server_local_path) | "None" | -| HTSGET_DATA_SERVER_SERVE_AT | See [`data_server_serve_at`](#data_server_serve_at) | "None" | -| HTSGET_DATA_SERVER_CORS_ALLOW_CREDENTIALS | See [`data_server_cors_allow_credentials`](#data_server_cors_allow_credentials) | "false" | -| HTSGET_DATA_SERVER_CORS_ALLOW_ORIGINS | See [`data_server_cors_allow_origins`](#data_server_cors_allow_origins) | "http://localhost:8081" | -| HTSGET_DATA_SERVER_CORS_ALLOW_HEADERS | See [`data_server_cors_allow_headers`](#data_server_cors_allow_headers) | "" | -| HTSGET_DATA_SERVER_CORS_MAX_AGE | See [`data_server_cors_max_age`](#data_server_cors_max_age) | | -| HTSGET_DATA_SERVER_CORS_EXPOSE_HEADERS | See [`data_server_cors_expose_headers`](#data_server_cors_expose_headers) | | -| HTSGET_ID | See [`id`](#id) | "None" | -| HTSGET_NAME | See [`name`](#name) | "None" | -| HTSGET_VERSION | See [`version`](#version) | "None" | -| HTSGET_ORGANIZATION_NAME | See [`organization_name`](#organization_name) | "None" | -| HTSGET_ORGANIZATION_URL | See [`organization_url`](#organization_url) | "None" | -| HTSGET_CONTACT_URL | See [`contact_url`](#contact_url) | "None" | -| HTSGET_DOCUMENTATION_URL | See [`documentation_url`](#documentation_url) | "None" | -| HTSGET_CREATED_AT | See [`created_at`](#created_at) | "None" | -| HTSGET_UPDATED_AT | See [`updated_at`](#updated_at) | "None" | -| HTSGET_ENVIRONMENT | See [`environment`](#environment) | "None" | -| HTSGET_RESOLVERS | See [resolvers](#resolvers) | | +| Variable | Description | +|-----------------------------------------------|-------------------------------------------------------------------------------------| +| `HTSGET_TICKET_SERVER_ADDR` | See [`ticket_server_addr`](#ticket_server_addr) | +| `HTSGET_TICKET_SERVER_CORS_ALLOW_CREDENTIALS` | See [`ticket_server_cors_allow_credentials`](#ticket_server_cors_allow_credentials) | +| `HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGINS` | See [`ticket_server_cors_allow_origins`](#ticket_server_cors_allow_origins) | +| `HTSGET_TICKET_SERVER_CORS_ALLOW_HEADERS` | See [`ticket_server_cors_allow_headers`](#ticket_server_cors_allow_headers) | +| `HTSGET_TICKET_SERVER_CORS_MAX_AGE` | See [`ticket_server_cors_max_age`](#ticket_server_cors_max_age) | +| `HTSGET_TICKET_SERVER_CORS_EXPOSE_HEADERS` | See [`ticket_server_cors_expose_headers`](#ticket_server_cors_expose_headers) | +| `HTSGET_DATA_SERVER_ADDR` | See [`data_server_addr`](#data_server_addr) | +| `HTSGET_DATA_SERVER_LOCAL_PATH` | See [`data_server_local_path`](#data_server_local_path) | +| `HTSGET_DATA_SERVER_SERVE_AT` | See [`data_server_serve_at`](#data_server_serve_at) | +| `HTSGET_DATA_SERVER_CORS_ALLOW_CREDENTIALS` | See [`data_server_cors_allow_credentials`](#data_server_cors_allow_credentials) | +| `HTSGET_DATA_SERVER_CORS_ALLOW_ORIGINS` | See [`data_server_cors_allow_origins`](#data_server_cors_allow_origins) | +| `HTSGET_DATA_SERVER_CORS_ALLOW_HEADERS` | See [`data_server_cors_allow_headers`](#data_server_cors_allow_headers) | +| `HTSGET_DATA_SERVER_CORS_MAX_AGE` | See [`data_server_cors_max_age`](#data_server_cors_max_age) | +| `HTSGET_DATA_SERVER_CORS_EXPOSE_HEADERS` | See [`data_server_cors_expose_headers`](#data_server_cors_expose_headers) | +| `HTSGET_ID` | See [`id`](#id) | +| `HTSGET_NAME` | See [`name`](#name) | +| `HTSGET_VERSION` | See [`version`](#version) | +| `HTSGET_ORGANIZATION_NAME` | See [`organization_name`](#organization_name) | +| `HTSGET_ORGANIZATION_URL` | See [`organization_url`](#organization_url) | +| `HTSGET_CONTACT_URL` | See [`contact_url`](#contact_url) | +| `HTSGET_DOCUMENTATION_URL` | See [`documentation_url`](#documentation_url) | +| `HTSGET_CREATED_AT` | See [`created_at`](#created_at) | +| `HTSGET_UPDATED_AT` | See [`updated_at`](#updated_at) | +| `HTSGET_ENVIRONMENT` | See [`environment`](#environment) | +| `HTSGET_RESOLVERS` | See [resolvers](#resolvers) | In order to use `HTSGET_RESOLVERS`, the entire resolver config array must be set. The nested array of resolvers structure can be set using name key and value pairs, for example: From ad096c8c4bdee136b72d73d2abaac455e87f5ef4 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 23 Dec 2022 15:27:32 +1100 Subject: [PATCH 45/45] docs: add missing environment variable options --- htsget-config/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/htsget-config/README.md b/htsget-config/README.md index f48fb0e12..e9e749bac 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -60,8 +60,8 @@ To configure the data server, set the following options: | `data_server_addr` | The address for the data server. | Socket address | `'127.0.0.1:8080'` | | `data_server_local_path` | The local path which the data server can access to serve files. | Filesystem path | `'data'` | | `data_server_serve_at` | The path which the data server will prefix to all response URLs for tickets. | URL path | `'/data'` | -| `data_server_key` | The path to the PEM formatted X.509 private key used by the data server. This is used to enable TLS with HTTPS. | Filesystem path | Not set | -| `data_server_cert` | The path to the PEM formatted X.509 certificate used by the data server. This is used to enable TLS with HTTPS. | Filesystem path | Not set | +| `data_server_key` | The path to the PEM formatted X.509 private key used by the data server. This is used to enable TLS with HTTPS. | Filesystem path | Not set | +| `data_server_cert` | The path to the PEM formatted X.509 certificate used by the data server. This is used to enable TLS with HTTPS. | Filesystem path | Not set | | `data_server_cors_allow_credentials` | Controls the CORS Access-Control-Allow-Credentials for the data server. | Boolean | `false` | | `data_server_cors_allow_origins` | Set the CORS Access-Control-Allow-Origin returned by the data server, this can be set to `All` to send a wildcard, `Mirror` to echo back the request sent by the client, or a specific array of origins. | `'All'`, `'Mirror'` or a array of origins | `['http://localhost:8080']` | | `data_server_cors_allow_headers` | Set the CORS Access-Control-Allow-Headers returned by the data server, this can be set to `All` to allow all headers, or a specific array of headers. | `'All'`, or a array of headers | `'All'` | @@ -251,14 +251,18 @@ The following environment variables - corresponding to the TOML config - are ava | `HTSGET_TICKET_SERVER_CORS_ALLOW_CREDENTIALS` | See [`ticket_server_cors_allow_credentials`](#ticket_server_cors_allow_credentials) | | `HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGINS` | See [`ticket_server_cors_allow_origins`](#ticket_server_cors_allow_origins) | | `HTSGET_TICKET_SERVER_CORS_ALLOW_HEADERS` | See [`ticket_server_cors_allow_headers`](#ticket_server_cors_allow_headers) | +| `HTSGET_TICKET_SERVER_CORS_ALLOW_METHODS` | See [`ticket_server_cors_allow_methods`](#ticket_server_cors_allow_methods) | | `HTSGET_TICKET_SERVER_CORS_MAX_AGE` | See [`ticket_server_cors_max_age`](#ticket_server_cors_max_age) | | `HTSGET_TICKET_SERVER_CORS_EXPOSE_HEADERS` | See [`ticket_server_cors_expose_headers`](#ticket_server_cors_expose_headers) | | `HTSGET_DATA_SERVER_ADDR` | See [`data_server_addr`](#data_server_addr) | | `HTSGET_DATA_SERVER_LOCAL_PATH` | See [`data_server_local_path`](#data_server_local_path) | | `HTSGET_DATA_SERVER_SERVE_AT` | See [`data_server_serve_at`](#data_server_serve_at) | +| `HTSGET_DATA_SERVER_KEY` | See [`data_server_key`](#data_server_key) | +| `HTSGET_DATA_SERVER_CERT` | See [`data_server_cert`](#data_server_cert) | | `HTSGET_DATA_SERVER_CORS_ALLOW_CREDENTIALS` | See [`data_server_cors_allow_credentials`](#data_server_cors_allow_credentials) | | `HTSGET_DATA_SERVER_CORS_ALLOW_ORIGINS` | See [`data_server_cors_allow_origins`](#data_server_cors_allow_origins) | | `HTSGET_DATA_SERVER_CORS_ALLOW_HEADERS` | See [`data_server_cors_allow_headers`](#data_server_cors_allow_headers) | +| `HTSGET_DATA_SERVER_CORS_ALLOW_METHODS` | See [`data_server_cors_allow_methods`](#data_server_cors_allow_methods) | | `HTSGET_DATA_SERVER_CORS_MAX_AGE` | See [`data_server_cors_max_age`](#data_server_cors_max_age) | | `HTSGET_DATA_SERVER_CORS_EXPOSE_HEADERS` | See [`data_server_cors_expose_headers`](#data_server_cors_expose_headers) | | `HTSGET_ID` | See [`id`](#id) |