chore: add org update script

Makefile

@@ -435,6 +435,10 @@ _clean_old_external_volumes:
 	( docker volume inspect ${COMPOSE_PROJECT_NAME}_products|grep /rpool/off/clones && docker volume rm ${COMPOSE_PROJECT_NAME}_products ) || true
 	( docker volume inspect ${COMPOSE_PROJECT_NAME}_product_images|grep /rpool/off/clones && docker volume rm ${COMPOSE_PROJECT_NAME}_product_images ) || true
 
+save_orgs_to_mongodb:
+	@echo "🥫 Saving exsiting orgs into MongoDB …"
+	${DOCKER_COMPOSE} run --rm backend perl -I/opt/product-opener/lib /opt/product-opener/scripts/migrations/2024_06_save_existing_orgs_to_mongodb.pl "/mnt/podata/orgs"
+
 #------------#
 # Production #
 #------------#

lib/ProductOpener/Data.pm

@@ -51,6 +51,7 @@ BEGIN {
 		&get_emb_codes_collection
 		&get_recent_changes_collection
 		&remove_documents_by_ids
+		&get_orgs_collection
 
 	);    # symbols to export on request
 	%EXPORT_TAGS = (all => [@EXPORT_OK]);
@@ -202,6 +203,10 @@ sub get_recent_changes_collection ($timeout = undef) {
 	return get_collection($mongodb, 'recent_changes', $timeout);
 }
 
+sub get_orgs_collection ($timeout = undef) {
+	return get_collection($mongodb, 'orgs', $timeout);
+}
+
 sub get_collection ($database, $collection, $timeout = undef) {
 	return get_mongodb_client($timeout)->get_database($database)->get_collection($collection);
 }

lib/ProductOpener/Orgs.pm

@@ -70,6 +70,7 @@ use ProductOpener::Mail qw/:all/;
 use ProductOpener::Lang qw/lang/;
 use ProductOpener::Display qw/:all/;
 use ProductOpener::Tags qw/canonicalize_tag_link/;
+use ProductOpener::Data qw/:all/;
 
 use CGI qw/:cgi :form escapeHTML/;
 use Encode;
@@ -165,6 +166,11 @@ sub store_org ($org_ref) {
 		# TODO: create org and its users in Odoo CRM
 	}
 
+	# Store to MongoDB
+	my $orgs_collection = get_orgs_collection();
+	$orgs_collection->replace_one({"org_id" => $org_ref->{org_id}}, $org_ref, {upsert => 1});
+
+	# Store to file
 	store("$BASE_DIRS{ORGS}/" . $org_ref->{org_id} . ".sto", $org_ref);
 
 	return;

scripts/migrations/2024_06_save_existing_orgs_to_mongodb.pl

@@ -0,0 +1,101 @@
+#!/usr/bin/perl -w
+
+# This file is part of Product Opener.
+#
+# Product Opener
+# Copyright (C) 2011-2023 Association Open Food Facts
+# Contact: [email protected]
+# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France
+#
+# Product Opener is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+use Modern::Perl '2017';
+use utf8;
+use Storable qw(lock_retrieve);
+use MongoDB;
+use Encode;
+use ProductOpener::Paths qw/%BASE_DIRS ensure_dir_created/;
+
+my $database = "off";
+my $collection = "orgs";
+
+my $orgs_collection = MongoDB::MongoClient->new->get_database($database)->get_collection($collection);
+
+my @orgs = ();
+
+my $start_dir = "$BASE_DIRS{ORGS}/";
+ensure_dir_created($BASE_DIRS{ORGS});
+
+sub retrieve {
+    my $file = shift;
+    if (!-e $file) {
+        return;
+    }
+    my $return = undef;
+    eval { $return = lock_retrieve($file); };
+    return $return;
+}
+
+sub find_orgs {
+    my ($dir, $code) = @_;
+    my $dh;
+
+    opendir $dh, "$dir" or die "could not open $dir directory: $!\n";
+    foreach my $file (sort readdir($dh)) {
+        chomp($file);
+        if ($file =~ /^([a-zA-Z_][a-zA-Z0-9_]*)\.sto/) {
+            push @orgs, [$code, $1];
+        } else {
+            next if $file =~ /\./;
+            if (-d "$dir/$file") {
+                find_orgs("$dir/$file", "$code$file");
+            }
+        }
+    }
+    closedir $dh or print "could not close $dir dir: $!\n";
+}
+
+sub main {
+    find_orgs($start_dir, '');
+
+    my $count = scalar @orgs;
+    my $i = 0;
+    my %codes = ();
+
+    print STDERR "$count organizations to update\n";
+
+    foreach my $code_rev_ref (@orgs) {
+        my ($code, $rev) = @$code_rev_ref;
+
+        my $org_ref = retrieve("$start_dir/$rev.sto") or print "not defined $start_dir/$rev.sto\n";
+
+        if (defined $org_ref) {
+            next if (defined $org_ref->{deleted} && $org_ref->{deleted} eq 'on');
+
+            $org_ref->{_id} = $code . "." . $rev;
+
+            my $return = $orgs_collection->replace_one({"_id" => $org_ref->{_id}}, $org_ref, {upsert => 1});
+            print STDERR "return $return\n";
+            $i++;
+            $codes{$code} = 1;
+        }
+    }
+
+    print STDERR "$count organizations to update - $i organizations not empty or deleted\n";
+    print STDERR "scalar keys codes: " . (scalar keys %codes) . "\n";
+}
+
+main();
+
+exit(0);