diff --git a/momento/src/commands/cloud_linter/dynamodb.rs b/momento/src/commands/cloud_linter/dynamodb.rs index 69df162..9d06bc5 100644 --- a/momento/src/commands/cloud_linter/dynamodb.rs +++ b/momento/src/commands/cloud_linter/dynamodb.rs @@ -164,33 +164,91 @@ impl ResourceWithMetrics for DynamoDbResource { } } +pub(crate) async fn append_ttl_to_appropriate_ddb_resources( + config: &SdkConfig, + mut resources: Vec, + limiter: Arc, +) -> Result, CliError> { + log::debug!("describing ttl for dynamodb tables"); + let ddb_client = aws_sdk_dynamodb::Client::new(config); + let describe_ddb_ttl_bar = + ProgressBar::new(resources.len() as u64).with_message("Describing Dynamo DB Ttl"); + describe_ddb_ttl_bar + .set_style(ProgressStyle::with_template(" {msg} {bar} {eta}").expect("invalid template")); + for resource in &mut resources { + describe_ddb_ttl_bar.inc(1); + match resource { + Resource::DynamoDb(r) => { + if r.resource_type == ResourceType::DynamoDbGsi { + continue; + } + let consumed_write_ops_index = r.metrics.iter().position(|p| { + p.name + .eq_ignore_ascii_case("consumedwritecapacityunits_sum") + }); + match consumed_write_ops_index { + Some(index) => { + let consumed_write_capacity = + r.metrics.get(index).expect("index should exist"); + let sum: f64 = consumed_write_capacity.values.iter().sum(); + // a basic heuristic around whether or not we care to check to see if a ttl exists on a ddb table. If the dynamodb table + // has less than 10 tps average, then we don't care to check if ttl is enabled or not. + if sum < 10.0 * 60.0 * 60.0 * 24.0 * 30.0 { + log::debug!("skipping ttl check for table {}", r.id); + continue; + } + log::debug!("querying ttl for table {}", r.id); + let ttl_enabled = + is_ddb_ttl_enabled(&ddb_client, &r.id, Arc::clone(&limiter)).await?; + r.metadata.ttl_enabled = ttl_enabled; + } + // we did not find that metric, and therefore we assume that there are no consumed capacity units, meaning we don't care to + // check for a ttl on the ddb table + None => { + continue; + } + } + } + Resource::ElastiCache(_) => { + continue; + } + }; + } + describe_ddb_ttl_bar.finish(); + Ok(resources) +} + pub(crate) async fn get_ddb_resources( config: &SdkConfig, limiter: Arc, ) -> Result, CliError> { let ddb_client = aws_sdk_dynamodb::Client::new(config); - let bar = ProgressBar::new_spinner().with_message("Listing Dynamo DB tables"); - bar.enable_steady_tick(Duration::from_millis(100)); + let list_ddb_tables_bar = ProgressBar::new_spinner().with_message("Listing Dynamo DB tables"); + list_ddb_tables_bar.enable_steady_tick(Duration::from_millis(100)); let table_names = list_table_names(&ddb_client, Arc::clone(&limiter)).await?; - bar.finish(); + list_ddb_tables_bar.finish(); - let bar = + let describe_ddb_tables_bar = ProgressBar::new(table_names.len() as u64).with_message("Describing Dynamo DB tables"); - bar.set_style(ProgressStyle::with_template(" {msg} {bar} {eta}").expect("invalid template")); - let mut resources = Vec::new(); + describe_ddb_tables_bar + .set_style(ProgressStyle::with_template(" {msg} {bar} {eta}").expect("invalid template")); + let mut ddb_resources: Vec = Vec::new(); for table_name in table_names { let instances = fetch_ddb_resources(&ddb_client, &table_name, Arc::clone(&limiter)).await?; - let wrapped_resources = instances - .into_iter() - .map(Resource::DynamoDb) - .collect::>(); - resources.extend(wrapped_resources); - bar.inc(1); + for instance in instances { + ddb_resources.push(instance); + } + describe_ddb_tables_bar.inc(1); } - bar.finish(); + describe_ddb_tables_bar.finish(); - Ok(resources) + let wrapped_resources = ddb_resources + .into_iter() + .map(Resource::DynamoDb) + .collect::>(); + + Ok(wrapped_resources) } async fn list_table_names( @@ -218,19 +276,11 @@ async fn list_table_names( Ok(table_names) } -async fn fetch_ddb_resources( +async fn is_ddb_ttl_enabled( ddb_client: &aws_sdk_dynamodb::Client, table_name: &str, limiter: Arc, -) -> Result, CliError> { - let region = ddb_client - .config() - .region() - .map(|r| r.as_ref()) - .ok_or(CliError { - msg: "No region configured for client".to_string(), - })?; - +) -> Result { let ttl = rate_limit(Arc::clone(&limiter), || async { ddb_client .describe_time_to_live() @@ -247,6 +297,21 @@ async fn fetch_ddb_resources( .. }) ); + Ok(ttl_enabled) +} + +async fn fetch_ddb_resources( + ddb_client: &aws_sdk_dynamodb::Client, + table_name: &str, + limiter: Arc, +) -> Result, CliError> { + let region = ddb_client + .config() + .region() + .map(|r| r.as_ref()) + .ok_or(CliError { + msg: "No region configured for client".to_string(), + })?; let description = rate_limit(Arc::clone(&limiter), || async { ddb_client @@ -309,7 +374,7 @@ async fn fetch_ddb_resources( billing_mode, gsi_count, item_count, - ttl_enabled, + ttl_enabled: false, is_global_table, lsi_count, table_class, diff --git a/momento/src/commands/cloud_linter/elasticache.rs b/momento/src/commands/cloud_linter/elasticache.rs index e9c6b50..a5c75ef 100644 --- a/momento/src/commands/cloud_linter/elasticache.rs +++ b/momento/src/commands/cloud_linter/elasticache.rs @@ -104,6 +104,7 @@ pub(crate) async fn get_elasticache_resources( config: &SdkConfig, limiter: Arc, ) -> Result, CliError> { + log::debug!("describing elasticache resources"); let region = config.region().map(|r| r.as_ref()).ok_or(CliError { msg: "No region configured for client".to_string(), })?; diff --git a/momento/src/commands/cloud_linter/linter_cli.rs b/momento/src/commands/cloud_linter/linter_cli.rs index ae9a8f0..d794e6b 100644 --- a/momento/src/commands/cloud_linter/linter_cli.rs +++ b/momento/src/commands/cloud_linter/linter_cli.rs @@ -18,6 +18,8 @@ use crate::commands::cloud_linter::resource::DataFormat; use crate::commands::cloud_linter::utils::check_aws_credentials; use crate::error::CliError; +use super::dynamodb::append_ttl_to_appropriate_ddb_resources; + pub async fn run_cloud_linter(region: String) -> Result<(), CliError> { let config = aws_config::defaults(BehaviorVersion::latest()) .region(Region::new(region)) @@ -28,17 +30,35 @@ pub async fn run_cloud_linter(region: String) -> Result<(), CliError> { let output_file_path = "linter_results.json.gz"; check_output_is_writable(output_file_path).await?; - let quota = - Quota::per_second(core::num::NonZeroU32::new(1).expect("should create non-zero quota")); - let limiter = Arc::new(RateLimiter::direct(quota)); + let control_plane_quota = Quota::per_second( + core::num::NonZeroU32::new(10).expect("should create non-zero control_plane_quota"), + ); + let control_plane_limiter = Arc::new(RateLimiter::direct(control_plane_quota)); + + let describe_ttl_quota = Quota::per_second( + core::num::NonZeroU32::new(1).expect("should create non-zero describe_ttl_quota"), + ); + let describe_ttl_limiter = Arc::new(RateLimiter::direct(describe_ttl_quota)); - let mut resources = get_ddb_resources(&config, Arc::clone(&limiter)).await?; + let metrics_quota = + Quota::per_second(core::num::NonZeroU32::new(20).expect("should create non-zero quota")); + let metrics_limiter = Arc::new(RateLimiter::direct(metrics_quota)); + + let mut resources = get_ddb_resources(&config, Arc::clone(&control_plane_limiter)).await?; let mut elasticache_resources = - get_elasticache_resources(&config, Arc::clone(&limiter)).await?; + get_elasticache_resources(&config, Arc::clone(&control_plane_limiter)).await?; resources.append(&mut elasticache_resources); - let resources = append_metrics_to_resources(&config, Arc::clone(&limiter), resources).await?; + let resources = + append_metrics_to_resources(&config, Arc::clone(&metrics_limiter), resources).await?; + + let resources = append_ttl_to_appropriate_ddb_resources( + &config, + resources, + Arc::clone(&describe_ttl_limiter), + ) + .await?; let data_format = DataFormat { resources }; diff --git a/momento/src/commands/cloud_linter/metrics.rs b/momento/src/commands/cloud_linter/metrics.rs index 9ca9141..20c5f4f 100644 --- a/momento/src/commands/cloud_linter/metrics.rs +++ b/momento/src/commands/cloud_linter/metrics.rs @@ -18,8 +18,8 @@ use crate::error::CliError; #[derive(Serialize, Deserialize)] pub(crate) struct Metric { - name: String, - values: Vec, + pub name: String, + pub values: Vec, } pub(crate) struct MetricTarget { diff --git a/momento/src/commands/cloud_linter/resource.rs b/momento/src/commands/cloud_linter/resource.rs index 7d8c9d1..78e3cd6 100644 --- a/momento/src/commands/cloud_linter/resource.rs +++ b/momento/src/commands/cloud_linter/resource.rs @@ -11,7 +11,7 @@ pub(crate) enum Resource { ElastiCache(ElastiCacheResource), } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, PartialEq)] pub(crate) enum ResourceType { #[serde(rename = "AWS::DynamoDB::GSI")] DynamoDbGsi,