From 1e7d4296840b1efbd960922fe895476df33b9d40 Mon Sep 17 00:00:00 2001 From: Simon Jenkins Date: Mon, 22 Aug 2022 20:31:19 -0700 Subject: [PATCH] Connect to Redis --- .env | 1 + .github/workflows/container-deploy.yml | 1 + conf/apache.conf | 1 + cpanfile | 1 + docker-compose.yml | 1 + .../dev-environment-quick-start-guide.md | 1 + lib/ProductOpener/Config2_docker.pm | 4 ++++ lib/ProductOpener/Config2_sample.pm | 3 +++ lib/ProductOpener/Products.pm | 15 +++++++++++++++ 9 files changed, 28 insertions(+) diff --git a/.env b/.env index b9ec3dc8e6865..01e427243b5f0 100644 --- a/.env +++ b/.env @@ -37,6 +37,7 @@ MONGODB_CACHE_SIZE=8 # GB MONGO_INITDB_ROOT_USERNAME=root MONGO_INITDB_ROOT_PASSWORD=test ROBOTOFF_URL=http://api:5500 # connect to Robotoff running in separate docker-compose deployment +REDIS_URL=searchredis:6379 # Redis runs in a separate docker-compose deployment GOOGLE_CLOUD_VISION_API_KEY= CROWDIN_PROJECT_IDENTIFIER= CROWDIN_PROJECT_KEY= diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 3dd2e4a54514e..6ddbebd210a07 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -113,6 +113,7 @@ jobs: echo "POSTGRES_USER=${{ secrets.POSTGRES_USER }}" >> .env echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .env echo "ROBOTOFF_URL=${{ secrets.ROBOTOFF_URL }}" >> .env + echo "REDIS_URL=${{ secrets.REDIS_URL }}" >> .env echo "GOOGLE_CLOUD_VISION_API_KEY=${{ secrets.GOOGLE_CLOUD_VISION_API_KEY }}" >> .env echo "CROWDIN_PROJECT_IDENTIFIER=${{ secrets.CROWDIN_PROJECT_IDENTIFIER }}" >> .env echo "CROWDIN_PROJECT_KEY=${{ secrets.CROWDIN_PROJECT_KEY }}" >> .env diff --git a/conf/apache.conf b/conf/apache.conf index 4146175056de9..78003c4f0b17c 100644 --- a/conf/apache.conf +++ b/conf/apache.conf @@ -14,6 +14,7 @@ PerlPassEnv PRODUCT_OPENER_DOMAIN PerlPassEnv PRODUCT_OPENER_PORT PerlPassEnv PRODUCERS_PLATFORM PerlPassEnv ROBOTOFF_URL +PerlPassEnv REDIS_URL PerlPassEnv MONGODB_HOST PerlPassEnv GOOGLE_CLOUD_VISION_API_KEY PerlPassEnv CROWDIN_PROJECT_IDENTIFIER diff --git a/cpanfile b/cpanfile index f04803ec70e98..250a92ea4c438 100644 --- a/cpanfile +++ b/cpanfile @@ -64,6 +64,7 @@ requires 'JSON::Create'; requires 'JSON::Parse'; requires 'Data::DeepAccess'; requires 'XML::XML2JSON'; +requires 'Redis::Client'; # Mojolicious/Minion diff --git a/docker-compose.yml b/docker-compose.yml index 2c2b47c154b34..8463b63a424bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,7 @@ x-backend-conf: &backend-conf - POSTGRES_USER - POSTGRES_PASSWORD - ROBOTOFF_URL + - REDIS_URL - GOOGLE_CLOUD_VISION_API_KEY - CROWDIN_PROJECT_IDENTIFIER - CROWDIN_PROJECT_KEY diff --git a/docs/introduction/dev-environment-quick-start-guide.md b/docs/introduction/dev-environment-quick-start-guide.md index e37fa3001612a..63a54e99e985a 100644 --- a/docs/introduction/dev-environment-quick-start-guide.md +++ b/docs/introduction/dev-environment-quick-start-guide.md @@ -77,6 +77,7 @@ The `.env` file contains ProductOpener default settings: | `PRODUCT_OPENER_FLAVOR_SHORT` | can be modified to run different flavors of OpenFoodFacts, amongst `off` (default), `obf`, `oppf`, `opf`.| | `PRODUCERS_PLATFORM` | can be set to `1` to build / run the **producer platform**.| | `ROBOTOFF_URL` | can be set to **connect with a Robotoff instance**.| +| `REDIS_URL` | can be set to **connect with a Redis instance for populating the search index**.| | `GOOGLE_CLOUD_VISION_API_KEY` | can be set to **enable OCR using Google Cloud Vision**.| | `CROWDIN_PROJECT_IDENTIFIER` and `CROWDIN_PROJECT_KEY` | can be set to **run translations**.| | `GEOLITE2_PATH`, `GEOLITE2_ACCOUNT_ID` and `GEOLITE2_LICENSE_KEY` | can be set to **enable Geolite2**.| diff --git a/lib/ProductOpener/Config2_docker.pm b/lib/ProductOpener/Config2_docker.pm index 61b8d8c3fa287..ff3f0441ea0fc 100755 --- a/lib/ProductOpener/Config2_docker.pm +++ b/lib/ProductOpener/Config2_docker.pm @@ -46,6 +46,7 @@ BEGIN $crowdin_project_identifier $crowdin_project_key $robotoff_url + $redis_url %server_options ); %EXPORT_TAGS = (all => [@EXPORT_OK]); @@ -93,6 +94,9 @@ my $postgres_url = "postgresql://${postgres_user}:${postgres_password}\@${postgr # enable an in-site robotoff-asker in the product page $robotoff_url = $ENV{ROBOTOFF_URL}; +# Set this to your instance of the search service to enable writes to it +$redis_url = $ENV{REDIS_URL}; + %server_options = ( private_products => $producers_platform, # 1 to make products visible only to the owner (producer platform) producers_platform => $producers_platform, diff --git a/lib/ProductOpener/Config2_sample.pm b/lib/ProductOpener/Config2_sample.pm index cac0732518598..61380416ff1af 100644 --- a/lib/ProductOpener/Config2_sample.pm +++ b/lib/ProductOpener/Config2_sample.pm @@ -40,6 +40,7 @@ BEGIN $crowdin_project_identifier $crowdin_project_key $robotoff_url + $redis_url %server_options @@ -74,6 +75,8 @@ $crowdin_project_key = ''; # enable an in-site robotoff-asker in the product page $robotoff_url = ''; +$redis_url = ''; + %server_options = ( cookie_domain => "openfoodfacts.dev", # if not set, default to $server _domain diff --git a/lib/ProductOpener/Products.pm b/lib/ProductOpener/Products.pm index 5c8f16bdb29b0..4c9759c6c7dbd 100644 --- a/lib/ProductOpener/Products.pm +++ b/lib/ProductOpener/Products.pm @@ -141,6 +141,8 @@ use Data::DeepAccess qw(deep_get); use LWP::UserAgent; use Storable qw(dclone); +use Redis::Client; + use Algorithm::CheckDigits; my $ean_check = CheckDigits('ean'); @@ -1175,6 +1177,9 @@ sub store_product($user_id, $product_ref, $comment) { store("$new_data_root/products/$path/changes.sto", $changes_ref); log_change($product_ref, $change_ref); + # index for search service + index_search_service($product_ref); + $log->debug("store_product - done", { code => $code, product_id => $product_id } ) if $log->is_debug(); return 1; @@ -2411,6 +2416,16 @@ sub index_product($product_ref) return; } +sub index_search_service($product_ref) { + # Now send the barcode to Redis so that it can be indexed + eval { + my $redis_client = Redis::Client->new(host => $ProductOpener::Config2::redis_url); + $redis_client->rpush('search_import_queue', $product_ref->{code}); + }; + if ($@) { + $log->warn("Error connecting to Redis", { error => $@ }) if $log->is_warn(); + } +} sub compute_codes($product_ref) {