diff --git a/.env b/.env index ab5f2e9914534..9666015ae3ec2 100644 --- a/.env +++ b/.env @@ -45,6 +45,9 @@ MONGODB_CACHE_SIZE=8 # GB MONGO_INITDB_ROOT_USERNAME=root MONGO_INITDB_ROOT_PASSWORD=test ROBOTOFF_URL=http://robotoff.openfoodfacts.localhost:5500 # connect to Robotoff running in separate docker-compose deployment +# connect to openfoodfacts-query running in separate docker-compose deployment. +# To test locally change to http://host.docker.internal:5510 +QUERY_URL=http://query:5510 EVENTS_URL= FACETS_KP_URL = https://facets-kp.openfoodfacts.org/render-to-html # use this to push products to openfoodfacts-search diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 67e74e422c253..8a35f9d4a51b8 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -29,6 +29,7 @@ jobs: echo "REDIS_URL=searchredis:6379" >> $GITHUB_ENV echo "MONGODB_HOST=10.1.0.200" >> $GITHUB_ENV echo "ROBOTOFF_URL=https://robotoff.openfoodfacts.net" >> $GITHUB_ENV + echo "QUERY_URL=http://10.1.0.200:5511" >> $GITHUB_ENV echo "PRODUCT_OPENER_DOMAIN=openfoodfacts.net" >> $GITHUB_ENV echo "PRODUCT_OPENER_FLAVOR=openfoodfacts" >> $GITHUB_ENV echo "PRODUCT_OPENER_FLAVOR_SHORT=off" >> $GITHUB_ENV @@ -144,6 +145,7 @@ jobs: echo "POSTGRES_USER=${{ env.POSTGRES_USER }}" >> .env echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .env echo "ROBOTOFF_URL=${{ env.ROBOTOFF_URL }}" >> .env + echo "QUERY_URL=${{ env.QUERY_URL }}" >> .env echo "REDIS_URL=${{ env.REDIS_URL }}" >> .env echo "FACETS_KP_URL=${{ env.FACETS_KP_URL }}" >> .env echo "GOOGLE_CLOUD_VISION_API_KEY=${{ secrets.GOOGLE_CLOUD_VISION_API_KEY }}" >> .env diff --git a/Makefile b/Makefile index e1273f5f88fe6..5e032d3705c62 100644 --- a/Makefile +++ b/Makefile @@ -201,6 +201,8 @@ refresh_product_tags: # get id for mongodb container docker cp scripts/refresh_products_tags.js $(shell docker-compose ps -q mongodb):/data/db ${DOCKER_COMPOSE} exec -T mongodb //bin/sh -c "mongo off /data/db/refresh_products_tags.js" + @echo "🥫 Refreshing product data cached in Postgres …" + ${DOCKER_COMPOSE} run --rm backend perl /opt/product-opener/scripts/refresh_postgres.pl ${from} import_sample_data: @echo "🥫 Importing sample data (~200 products) into MongoDB …" diff --git a/conf/apache.conf b/conf/apache.conf index 573c43cc4f697..0485e3952b45f 100644 --- a/conf/apache.conf +++ b/conf/apache.conf @@ -14,6 +14,7 @@ PerlPassEnv PRODUCT_OPENER_DOMAIN PerlPassEnv PRODUCT_OPENER_PORT PerlPassEnv PRODUCERS_PLATFORM PerlPassEnv ROBOTOFF_URL +PerlPassEnv QUERY_URL PerlPassEnv EVENTS_URL PerlPassEnv FACETS_KP_URL PerlPassEnv EVENTS_USERNAME diff --git a/docker-compose.yml b/docker-compose.yml index 14ee342a43b11..ef17f87daadc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ x-backend-conf: &backend-conf - POSTGRES_USER - POSTGRES_PASSWORD - ROBOTOFF_URL + - QUERY_URL - EVENTS_URL - FACETS_KP_URL - EVENTS_USERNAME diff --git a/docker/dev.yml b/docker/dev.yml index 27446ee5295d4..804d2105d34aa 100644 --- a/docker/dev.yml +++ b/docker/dev.yml @@ -20,6 +20,11 @@ x-backend-conf: &backend-conf - ./docker/docker-entrypoint.sh:/docker-entrypoint.sh # we use this for debugging, from times to times - ./debug:/mnt/podata/debug/ + # Allow the container to connect to the host when using Linux. + # This is needed to access other services that are running outside of Docker + # e.g. when developing with Robotoff or openfoodfacts-query + extra_hosts: + - "host.docker.internal:host-gateway" x-minion-db-network: &minion-db-network networks: diff --git a/docs/dev/how-to-quick-start-guide.md b/docs/dev/how-to-quick-start-guide.md index 12b9b0e9438aa..0d7d4b47cf1dd 100644 --- a/docs/dev/how-to-quick-start-guide.md +++ b/docs/dev/how-to-quick-start-guide.md @@ -96,6 +96,7 @@ The `.env` file contains ProductOpener default settings: | `PRODUCT_OPENER_FLAVOR_SHORT` | can be modified to run different flavors of OpenFoodFacts, amongst `off` (default), `obf`, `oppf`, `opf`.| | `PRODUCERS_PLATFORM` | can be set to `1` to build / run the **producer platform**.| | `ROBOTOFF_URL` | can be set to **connect with a Robotoff instance**.| +| `QUERY_URL` | can be set to **connect with a Query instance**.| | `REDIS_URL` | can be set to **connect with a Redis instance for populating the search index**.| | `GOOGLE_CLOUD_VISION_API_KEY` | can be set to **enable OCR using Google Cloud Vision**.| | `CROWDIN_PROJECT_IDENTIFIER` and `CROWDIN_PROJECT_KEY` | can be set to **run translations**.| diff --git a/lib/ProductOpener/Config2_docker.pm b/lib/ProductOpener/Config2_docker.pm index f822c1ef44eed..a711ea7a11245 100755 --- a/lib/ProductOpener/Config2_docker.pm +++ b/lib/ProductOpener/Config2_docker.pm @@ -47,6 +47,7 @@ BEGIN { $crowdin_project_identifier $crowdin_project_key $robotoff_url + $query_url $events_url $facets_kp_url $events_username @@ -105,6 +106,10 @@ $log_emails = $ENV{OFF_LOG_EMAILS} // 0; # enable an in-site robotoff-asker in the product page $robotoff_url = $ENV{ROBOTOFF_URL}; +# Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-query/ to +# enable product counts and aggregations / facets +$query_url = $ENV{QUERY_URL}; + # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events # enable creating events for some actions (e.g. when a product is edited) $events_url = $ENV{EVENTS_URL}; diff --git a/lib/ProductOpener/Config2_sample.pm b/lib/ProductOpener/Config2_sample.pm index 8dc64184c2034..fe08108ff9da0 100644 --- a/lib/ProductOpener/Config2_sample.pm +++ b/lib/ProductOpener/Config2_sample.pm @@ -39,6 +39,7 @@ BEGIN { $crowdin_project_identifier $crowdin_project_key $robotoff_url + $query_url $events_url $events_username $events_password @@ -75,6 +76,7 @@ $crowdin_project_key = ''; # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to # enable an in-site robotoff-asker in the product page $robotoff_url = ''; +$query_url = ''; # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events # enable creating events for some actions (e.g. when a product is edited) diff --git a/lib/ProductOpener/Config_obf.pm b/lib/ProductOpener/Config_obf.pm index b57fe0d23f058..dd41b2d3f1471 100644 --- a/lib/ProductOpener/Config_obf.pm +++ b/lib/ProductOpener/Config_obf.pm @@ -49,6 +49,7 @@ BEGIN { $crowdin_project_key $robotoff_url + $query_url $events_url $events_username $events_password @@ -206,6 +207,7 @@ $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key; # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to # enable an in-site robotoff-asker in the product page $robotoff_url = $ProductOpener::Config2::robotoff_url; +$query_url = $ProductOpener::Config2::query_url; # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events # enable creating events for some actions (e.g. when a product is edited) diff --git a/lib/ProductOpener/Config_off.pm b/lib/ProductOpener/Config_off.pm index 20de113f72f64..985c014d5227f 100644 --- a/lib/ProductOpener/Config_off.pm +++ b/lib/ProductOpener/Config_off.pm @@ -49,6 +49,7 @@ BEGIN { $log_emails $robotoff_url + $query_url $events_url $events_username $events_password @@ -354,6 +355,7 @@ $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key; # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to # enable an in-site robotoff-asker in the product page $robotoff_url = $ProductOpener::Config2::robotoff_url; +$query_url = $ProductOpener::Config2::query_url; # do we want to send emails $log_emails = $ProductOpener::Config2::log_emails; @@ -461,7 +463,6 @@ my $manifest = { }; $options{manifest} = $manifest; -$options{mongodb_supports_sample} = 0; # from MongoDB 3.2 onward $options{display_random_sample_of_products_after_edits} = 0; # from MongoDB 3.2 onward $options{favicons} = <run(); } +sub execute_aggregate_tags_query ($aggregate_parameters) { + return execute_tags_query('aggregate', $aggregate_parameters); +} + +sub execute_count_tags_query ($query_ref) { + return execute_tags_query('count', $query_ref); +} + +sub execute_tags_query ($type, $parameters) { + if ((defined $query_url) and (length($query_url) > 0)) { + $query_url =~ s/^\s+|\s+$//g; + my $path = "$query_url/$type"; + $log->debug('Executing PostgreSQL ' . $type . ' query on ' . $path, {query => $parameters}) + if $log->is_debug(); + + my $ua = LWP::UserAgent->new(); + my $resp = $ua->post( + $path, + Content => encode_json($parameters), + 'Content-Type' => 'application/json; charset=utf-8' + ); + if ($resp->is_success) { + return decode_json($resp->decoded_content); + } + else { + $log->warn( + "query response not ok", + { + code => $resp->code, + status_line => $resp->status_line, + response => $resp + } + ) if $log->is_warn(); + return; + } + } + else { + $log->debug('QUERY_URL not defined') if $log->is_debug(); + return; + } +} + =head2 get_products_collection( $parameters_ref ) C establishes a connection to MongoDB and uses timeout as an argument. This then selects a collection diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index d58b4a92bdd9f..e836c72774c3c 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -201,6 +201,7 @@ use Template; use Devel::Size qw(size total_size); use Data::DeepAccess qw(deep_get); use Log::Log4perl; +use LWP::UserAgent; use Log::Any '$log', default_adapter => 'Stderr'; @@ -1509,6 +1510,11 @@ sub set_cache_results ($key, $results) { return; } +sub can_use_query_cache() { + return ( ((not defined single_param("no_cache")) or (not single_param("no_cache"))) + and (not $server_options{producers_platform})); +} + sub query_list_of_tags ($request_ref, $query_ref) { add_country_and_owner_filters_to_query($request_ref, $query_ref); @@ -1605,13 +1611,41 @@ sub query_list_of_tags ($request_ref, $query_ref) { my $results = get_cache_results($key, $request_ref); if ((not defined $results) or (ref($results) ne "ARRAY") or (not defined $results->[0])) { - - # do not use the smaller cached products_tags collection if ?no_cache=1 + $results = undef; + # do not use the postgres cache if ?no_cache=1 # or if we are on the producers platform - if ( ((defined single_param("no_cache")) and (single_param("no_cache"))) - or ($server_options{producers_platform})) - { + if (can_use_query_cache()) { + $results = execute_aggregate_tags_query($aggregate_parameters); + + if (not defined $results) { + eval { + $log->debug("Executing MongoDB aggregate query on products_tags collection", + {query => $aggregate_parameters}) + if $log->is_debug(); + $results = execute_query( + sub { + return get_products_collection( + get_products_collection_request_parameters($request_ref, {tags => 1})) + ->aggregate($aggregate_parameters, {allowDiskUse => 1}); + } + ); + # the return value of aggregate has changed from version 0.702 + # and v1.4.5 of the perl MongoDB module + $results = [$results->all] if defined $results; + }; + my $err = $@; + if ($err) { + $log->warn("MongoDB error", {error => $err}) if $log->is_warn(); + } + else { + $log->info("MongoDB query ok", {error => $err}) if $log->is_info(); + } + $log->debug("MongoDB query done", {error => $err}) if $log->is_debug(); + } + } + + if (not defined $results) { eval { $log->debug("Executing MongoDB aggregate query on products collection", {query => $aggregate_parameters}) @@ -1622,39 +1656,24 @@ sub query_list_of_tags ($request_ref, $query_ref) { ->aggregate($aggregate_parameters, {allowDiskUse => 1}); } ); + # the return value of aggregate has changed from version 0.702 + # and v1.4.5 of the perl MongoDB module + $results = [$results->all] if defined $results; }; - } - else { + my $err = $@; + if ($err) { + $log->warn("MongoDB error", {error => $err}) if $log->is_warn(); + } + else { + $log->info("MongoDB query ok", {error => $err}) if $log->is_info(); + } - eval { - $log->debug("Executing MongoDB aggregate query on products_tags collection", - {query => $aggregate_parameters}) - if $log->is_debug(); - $results = execute_query( - sub { - return get_products_collection( - get_products_collection_request_parameters($request_ref, {tags => 1})) - ->aggregate($aggregate_parameters, {allowDiskUse => 1}); - } - ); - }; - } - if ($@) { - $log->warn("MongoDB error", {error => $@}) if $log->is_warn(); - } - else { - $log->info("MongoDB query ok", {error => $@}) if $log->is_info(); + $log->debug("MongoDB query done", {error => $err}) if $log->is_debug(); } - $log->debug("MongoDB query done", {error => $@}) if $log->is_debug(); - $log->trace("aggregate query done") if $log->is_trace(); - # the return value of aggregate has changed from version 0.702 - # and v1.4.5 of the perl MongoDB module if (defined $results) { - $results = [$results->all]; - if (defined $results->[0] and $cache_results_flag) { set_cache_results($key, $results); } @@ -1689,41 +1708,43 @@ sub query_list_of_tags ($request_ref, $query_ref) { if (not defined $results_count) { my $count_results; - - # do not use the smaller cached products_tags collection if ?no_cache=1 + # do not use the smaller postgres cache if ?no_cache=1 # or if we are on the producers platform - if ( ((defined single_param("no_cache")) and (single_param("no_cache"))) - or ($server_options{producers_platform})) - { - eval { - $log->debug("Executing MongoDB aggregate count query on products collection", - {query => $aggregate_count_parameters}) - if $log->is_debug(); - $count_results = execute_query( - sub { - return get_products_collection(get_products_collection_request_parameters($request_ref)) - ->aggregate($aggregate_count_parameters, {allowDiskUse => 1}); - } - ); + if (can_use_query_cache()) { + $count_results = execute_aggregate_tags_query($aggregate_count_parameters); + + if (not defined $count_results) { + eval { + $log->debug("Executing MongoDB aggregate count query on products_tags collection", + {query => $aggregate_count_parameters}) + if $log->is_debug(); + $count_results = execute_query( + sub { + return get_products_collection( + get_products_collection_request_parameters($request_ref, {tags => 1})) + ->aggregate($aggregate_count_parameters, {allowDiskUse => 1}); + } + ); + } } } - else { + + if (not defined $count_results) { eval { - $log->debug("Executing MongoDB aggregate count query on products_tags collection", + $log->debug("Executing MongoDB aggregate count query on products collection", {query => $aggregate_count_parameters}) if $log->is_debug(); $count_results = execute_query( sub { - return get_products_collection( - get_products_collection_request_parameters($request_ref, {tags => 1})) + return get_products_collection(get_products_collection_request_parameters($request_ref)) ->aggregate($aggregate_count_parameters, {allowDiskUse => 1}); } ); + $count_results = [$count_results->all]->[0] if defined $count_results; } } - if ((not $@) and (defined $count_results)) { - $count_results = [$count_results->all]->[0]; + if (defined $count_results) { $request_ref->{structured_response}{count} = $count_results->{$groupby_tagtype . "_tags"}; if ($cache_results_flag) { @@ -5210,133 +5231,18 @@ sub search_and_display_products ($request_ref, $query_ref, $sort_by, $limit, $pa my $cursor; eval { - if (($options{mongodb_supports_sample}) and (defined $request_ref->{sample_size})) { - $log->debug("Counting MongoDB documents for query", {query => $query_ref}) if $log->is_debug(); - $count = execute_query( - sub { - return get_products_collection( - get_products_collection_request_parameters($request_ref, {tags => 1})) - ->count_documents($query_ref); - } - ); - $log->info("MongoDB count query ok", {error => $@, count => $count}) if $log->is_info(); - - my $aggregate_parameters - = [{"\$match" => $query_ref}, {"\$sample" => {"size" => $request_ref->{sample_size}}}]; - $log->debug("Executing MongoDB query", {query => $aggregate_parameters}) if $log->is_debug(); - $cursor = execute_query( - sub { - return get_products_collection( - get_products_collection_request_parameters($request_ref, {tags => 1})) - ->aggregate($aggregate_parameters, {allowDiskUse => 1}); - } - ); - } - else { - $log->debug("Counting MongoDB documents for query", {query => $query_ref}) if $log->is_debug(); - # test if query_ref is empty - if (single_param('no_count')) { - # Skip the count if it is not needed - # e.g. for some API queries - $log->debug("no_count is set, skipping count") if $log->is_debug(); - } - elsif (keys %{$query_ref} > 0) { - #check if count results is in cache - my $key_count = generate_cache_key("search_products_count", $query_ref); - $log->debug("MongoDB query key - search_products_count", {key => $key_count}) if $log->is_debug(); - my $results_count = get_cache_results($key_count, $request_ref); - if (not defined $results_count) { - - $log->debug("count not in cache for query", {key => $key_count}) if $log->is_debug(); - - # Count queries are very expensive, if possible, execute them on the smaller products_tags collection - my $only_tags_filters = 1; - - if ($server_options{producers_platform}) { - $only_tags_filters = 0; - } - else { - - foreach my $field (keys %$query_ref) { - if ($field !~ /_tags$/) { - $log->debug( - "non tags field in query filters, cannot use smaller products_tags collection", - {field => $field, value => $query_ref->{field}} - ) if $log->is_debug(); - $only_tags_filters = 0; - last; - } - } - } - - if ( ($only_tags_filters) - and ((not defined single_param("no_cache")) or (single_param("no_cache") == 0))) - { + $count = estimate_result_count($request_ref, $query_ref, $cache_results_flag); - $count = execute_query( - sub { - $log->debug("count_documents on smaller products_tags collection", - {key => $key_count}) - if $log->is_debug(); - return get_products_collection( - get_products_collection_request_parameters($request_ref, {tags => 1})) - ->count_documents($query_ref); - } - ); - - } - else { - - $count = execute_query( - sub { - $log->debug("count_documents on complete products collection", {key => $key_count}) - if $log->is_debug(); - return get_products_collection( - get_products_collection_request_parameters($request_ref)) - ->count_documents($query_ref); - } - ); - } - if ($@) { - $log->warn("MongoDB error during count", {error => $@}) if $log->is_warn(); - } - elsif ($cache_results_flag) { - $log->debug("count query complete, setting cache", {key => $key_count, count => $count}) - if $log->is_debug(); - set_cache_results($key_count, $count); - } - } - else { - # Cached result - $count = $results_count; - $log->debug("count in cache for query", {key => $key_count, count => $count}) - if $log->is_debug(); - } - } - else { - # if query_ref is empty (root URL world.openfoodfacts.org) use estimated_document_count for better performance - $count = execute_query( - sub { - $log->debug("empty query_ref, use estimated_document_count fot better performance", {}) - if $log->is_debug(); - return get_products_collection(get_products_collection_request_parameters($request_ref)) - ->estimated_document_count(); - } - ); + $log->debug("Executing MongoDB query", + {query => $query_ref, fields => $fields_ref, sort => $sort_ref, limit => $limit, skip => $skip}) + if $log->is_debug(); + $cursor = execute_query( + sub { + return get_products_collection(get_products_collection_request_parameters($request_ref)) + ->query($query_ref)->fields($fields_ref)->sort($sort_ref)->limit($limit)->skip($skip); } - $log->info("MongoDB count query done", {error => $@, count => $count}) if $log->is_info(); - - $log->debug("Executing MongoDB query", - {query => $query_ref, fields => $fields_ref, sort => $sort_ref, limit => $limit, skip => $skip}) - if $log->is_debug(); - $cursor = execute_query( - sub { - return get_products_collection(get_products_collection_request_parameters($request_ref)) - ->query($query_ref)->fields($fields_ref)->sort($sort_ref)->limit($limit)->skip($skip); - } - ); - $log->info("MongoDB query ok", {error => $@}) if $log->is_info(); - } + ); + $log->info("MongoDB query ok", {error => $@}) if $log->is_info(); }; if ($@) { $log->warn("MongoDB error", {error => $@}) if $log->is_warn(); @@ -5616,6 +5522,116 @@ JS return $html; } +sub estimate_result_count ($request_ref, $query_ref, $cache_results_flag) { + my $count; + my $err; + + $log->debug("Counting MongoDB documents for query", {query => $query_ref}) if $log->is_debug(); + # test if query_ref is empty + if (single_param('no_count')) { + # Skip the count if it is not needed + # e.g. for some API queries + $log->debug("no_count is set, skipping count") if $log->is_debug(); + } + elsif (keys %{$query_ref} > 0) { + #check if count results is in cache + my $key_count = generate_cache_key("search_products_count", $query_ref); + $log->debug("MongoDB query key - search_products_count", {key => $key_count}) if $log->is_debug(); + $count = get_cache_results($key_count, $request_ref); + if (not defined $count) { + + $log->debug("count not in cache for query", {key => $key_count}) if $log->is_debug(); + + # Count queries are very expensive, if possible, execute them on the postgres cache + if (can_use_query_cache()) { + $count = execute_count_tags_query($query_ref); + + if (not defined $count) { + # Count queries are very expensive, if possible, execute them on the smaller products_tags collection + my $only_tags_filters = 1; + + if ($server_options{producers_platform}) { + $only_tags_filters = 0; + } + else { + + foreach my $field (keys %$query_ref) { + if ($field !~ /_tags$/) { + $log->debug( + "non tags field in query filters, cannot use smaller products_tags collection", + {field => $field, value => $query_ref->{field}}) + if $log->is_debug(); + $only_tags_filters = 0; + last; + } + } + } + + if ( ($only_tags_filters) + and ((not defined single_param("no_cache")) or (single_param("no_cache") == 0))) + { + + $count = execute_query( + sub { + $log->debug("count_documents on smaller products_tags collection", {key => $key_count}) + if $log->is_debug(); + return get_products_collection( + get_products_collection_request_parameters($request_ref, {tags => 1})) + ->count_documents($query_ref); + } + ); + $err = $@; + if ($err) { + $log->warn("MongoDB error during count", {error => $err}) if $log->is_warn(); + } + } + } + } + + if (not defined $count) { + $count = execute_query( + sub { + $log->debug("count_documents on complete products collection", {key => $key_count}) + if $log->is_debug(); + return get_products_collection(get_products_collection_request_parameters($request_ref)) + ->count_documents($query_ref); + } + ); + $err = $@; + if ($err) { + $log->warn("MongoDB error during count", {error => $err}) if $log->is_warn(); + } + } + + if ((defined $count) and $cache_results_flag) { + $log->debug("count query complete, setting cache", {key => $key_count, count => $count}) + if $log->is_debug(); + set_cache_results($key_count, $count); + } + } + else { + # Cached result + $log->debug("count in cache for query", {key => $key_count, count => $count}) + if $log->is_debug(); + } + } + else { + # if query_ref is empty (root URL world.openfoodfacts.org) use estimated_document_count for better performance + $count = execute_query( + sub { + $log->debug("empty query_ref, use estimated_document_count fot better performance", {}) + if $log->is_debug(); + return get_products_collection(get_products_collection_request_parameters($request_ref)) + ->estimated_document_count(); + } + ); + $err = $@; + } + $log->info("Count query done", {error => $err, count => $count}) if $log->is_info(); + + return $count; +} + =head2 display_pagination( $request_ref , $count , $limit , $page ) This function is used for page navigation and gets called when there is more diff --git a/scripts/refresh_postgres.pl b/scripts/refresh_postgres.pl new file mode 100644 index 0000000000000..650d40e24493f --- /dev/null +++ b/scripts/refresh_postgres.pl @@ -0,0 +1,40 @@ +#!/usr/bin/perl -w + +# This file is part of Product Opener. +# +# Product Opener +# Copyright (C) 2011-2023 Association Open Food Facts +# Contact: contact@openfoodfacts.org +# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France +# +# Product Opener is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +use Modern::Perl '2017'; + +use ProductOpener::Config qw/:all/; + +use LWP::UserAgent; + +if (defined $query_url) { + $query_url =~ s/^\s+|\s+$//g; + my $from = $ARGV[0] || ''; + my $path = "$query_url/importfrommongo?from=$from"; + print STDERR "Importing to $path\n"; + my $ua = LWP::UserAgent->new(); + $ua->get($path); + print STDERR "Finished\n"; +} +else { + print STDERR "QUERY_URL not defined\n"; +}