From 433af1dc512c797be633f0e4963b0c95d9a0fc5d Mon Sep 17 00:00:00 2001 From: Rouven Hi Date: Sat, 12 Nov 2022 11:46:58 +0100 Subject: [PATCH] feat(cache): Refactor resources endpoint and test pipeline env vars --- .github/workflows/build-image.yaml | 12 ++++++++++++ src/integration_test_resources_api.rs | 10 +++++++--- src/resource_endpoint.rs | 18 +++++++++--------- src/resource_processor.rs | 18 ------------------ src/resource_store.rs | 19 +++++++++++++++---- 5 files changed, 43 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 7a14596..1746faf 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -12,6 +12,18 @@ env: jobs: + set-image_tag: + name: Set container image tag + runs-on: ubuntu-latest + steps: + - name: Set image tag global + run: | + GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} + if [ $GIT_BRANCH = "main" ]; then export TEST_IMAGE_TAG=latest; else export TEST_IMAGE_TAG=next; fi + echo "TEST_IMAGE_TAG=$TEST_IMAGE_TAG" +# echo "TEST_IMAGE_TAG=1234" >> $GITHUB_ENV + env + check-oci-config: name: Check Containerfile runs-on: ubuntu-latest diff --git a/src/integration_test_resources_api.rs b/src/integration_test_resources_api.rs index 67a4f6d..75c01b2 100644 --- a/src/integration_test_resources_api.rs +++ b/src/integration_test_resources_api.rs @@ -2,8 +2,8 @@ use std::fs::File; use std::io::Read; use std::path::{Path, PathBuf}; -use std::{env, fs}; use std::ops::Add; +use std::{env, fs}; use actix_web::dev::{ServiceFactory, ServiceRequest, ServiceResponse}; use actix_web::{test, web, App, Error}; @@ -72,14 +72,18 @@ async fn test_this_week_in_past_resources() { TEST_JPEG_URL, ) .await; - let another_date_string = Local::now().date().add(Duration::weeks(4)).format("%Y%m%d").to_string(); + let another_date_string = Local::now() + .date() + .add(Duration::weeks(4)) + .format("%Y%m%d") + .to_string(); let _ = create_test_image( &base_test_dir, "", format!("IMG_{}.jpg", another_date_string).as_str(), TEST_JPEG_URL, ) - .await; + .await; // AND a running this-week-in-past instance let app_server = test::init_service(build_app(base_test_dir.to_str().unwrap())).await; diff --git a/src/resource_endpoint.rs b/src/resource_endpoint.rs index 189f8d6..1c2c87e 100644 --- a/src/resource_endpoint.rs +++ b/src/resource_endpoint.rs @@ -19,11 +19,13 @@ pub async fn get_all_resources(resource_store: web::Data) -> Http #[get("week")] pub async fn get_this_week_resources(resource_store: web::Data) -> HttpResponse { - let keys: Vec = resource_processor::get_this_week_in_past(resource_store.as_ref()); + let resource_ids: Vec = resource_store + .as_ref() + .get_resource_this_week_visible_random(); HttpResponse::Ok() .content_type("application/json") - .body(serde_json::to_string(&keys).unwrap()) + .body(serde_json::to_string(&resource_ids).unwrap()) } #[get("random")] @@ -51,7 +53,7 @@ pub async fn get_resource_by_id_and_resolution( let display_width = path_params.1; let display_height = path_params.2; - // Check cache, if successful return early + // Check cache, if successful return it let cached_data = resource_store .get_ref() .get_data_cache_entry(format!("{resource_id}_{display_width}_{display_height}")); @@ -64,13 +66,10 @@ pub async fn get_resource_by_id_and_resolution( // if not in cache, load resource metadata from database let remote_resource: Option = resource_store .get_resource(resource_id) - .and_then(|resource_json_string| { - serde_json::from_str(resource_json_string.as_str()) - .ok() - }); - // If we can't find the requested resource by id, get out here + .and_then(|resource_json_string| serde_json::from_str(resource_json_string.as_str()).ok()); + // If we can't find the requested resource by id, return with an error if remote_resource.is_none() { - return HttpResponse::InternalServerError().finish(); + return HttpResponse::NotFound().finish(); } // If we found the requested resource, read the image data and adjust the image to the display @@ -88,6 +87,7 @@ pub async fn get_resource_by_id_and_resolution( }, ); + // If image adjustments were successful, return the data, otherwise return with error if let Some(resource_data) = resource_data { resource_store.get_ref().add_data_cache_entry( format!("{resource_id}_{display_width}_{display_height}"), diff --git a/src/resource_processor.rs b/src/resource_processor.rs index eefd3f2..a89d62b 100644 --- a/src/resource_processor.rs +++ b/src/resource_processor.rs @@ -1,27 +1,9 @@ use std::env; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; - use crate::geo_location; use crate::resource_reader::RemoteResource; use crate::resource_store::ResourceStore; -/// Returns resources that was taken this week in the past -/// The resources are shuffled, to the result is not deterministic -/// Excluded are hidden resources -pub fn get_this_week_in_past(resource_store: &ResourceStore) -> Vec { - resource_store - .get_all_visible_resource_random() - .par_iter() - .map(|resource_json_string| { - serde_json::from_str::(resource_json_string.as_str()) - .unwrap_or_else(|_| panic!("Parsing of '{resource_json_string}' failed!")) - }) - .filter(|resource| resource.is_this_week()) - .map(|resource| resource.id) - .collect() -} - /// Builds the display value for the specified resource /// The display value contains the date and location of a resource pub async fn build_display_value( diff --git a/src/resource_store.rs b/src/resource_store.rs index cd9489d..43f3a72 100644 --- a/src/resource_store.rs +++ b/src/resource_store.rs @@ -25,11 +25,22 @@ impl ResourceStore { ids } - /// Gets a list of all visible resources in a random order - /// Returns a list of resource data - pub fn get_all_visible_resource_random(&self) -> Vec { + /// Gets a list of all visible resources this week in a random order + /// Returns a list of resource IDs + pub fn get_resource_this_week_visible_random(&self) -> Vec { let connection = self.persistent_file_store_pool.get().unwrap(); - let mut stmt = connection.prepare("SELECT value FROM resources WHERE id NOT IN (SELECT id FROM hidden) ORDER BY RANDOM();").unwrap(); + let mut stmt = connection + .prepare( + r#"SELECT DISTINCT resources.id + FROM resources, + json_each(resources.value) json + WHERE json.key = 'taken' + AND json.value NOT NULL + AND strftime('%W', json.value) = strftime('%W', 'now') + AND resources.id NOT IN (SELECT id FROM hidden) + ORDER BY RANDOM();"#, + ) + .unwrap(); let mut rows = stmt.query([]).unwrap(); let mut ids: Vec = Vec::new(); while let Ok(Some(row)) = rows.next() {