From 2b49db04ea0c5561ca26c760deba2a875d03fb32 Mon Sep 17 00:00:00 2001
From: Pierre Slamich <pierre@openfoodfacts.org>
Date: Tue, 19 Dec 2023 16:42:01 +0100
Subject: [PATCH] fix: more CIQUAL matches and translations (#9538)

---
 taxonomies/ingredients.txt                    | 146 ++++++++++++++++--
 .../ingredients/fi-do-not-match-myanmar.json  |   6 +-
 2 files changed, 134 insertions(+), 18 deletions(-)

diff --git a/taxonomies/ingredients.txt b/taxonomies/ingredients.txt
index 8195c6ceeb93e..7291a3ac2b21c 100644
--- a/taxonomies/ingredients.txt
+++ b/taxonomies/ingredients.txt
@@ -3239,6 +3239,11 @@ it:latte crudo
 ja:生乳
 nl:rauwe melk
 tr:çiğ süt, pastörize edilmemiş süt
+pl:surowe mleko
+pt:leite cru
+ro:lapte crud
+ru:сырое молоко
+zh:生牛奶
 wikipedia:en:https://en.wikipedia.org/wiki/Raw_milk
 
 <en:raw milk
@@ -3246,6 +3251,12 @@ wikipedia:en:https://en.wikipedia.org/wiki/Raw_milk
 fr:lait cru entier
 ca:llet crua sencera
 es:leche cruda entera
+it:latte crudo intero
+lt:neapdorotas pilnas pienas
+nl:rauwe volle melk
+pl:surowe mleko pełne
+pt:leite cru integral
+ro:lapte crud integral
 
 ################################## COW MILKS ###################################
 
@@ -3380,6 +3391,8 @@ tr:pastörize inek sütü
 fr:lait de vache thermisé
 ca:llet de vaca termitzada, llet termitzada de vaca
 es:leche de vaca termizada, leche termizada de vaca
+it:latte di mucca termizzato
+nl:gestremde koeienmelk
 
 <en:Cow's milk
 fr:lait frais de nos vaches non homogeneise
@@ -3529,6 +3542,13 @@ ro:lapte de capră pasteurizat, lapte de capra pasteurizat
 <en:pasteurised goat milk
 en:whole goat milk UHT
 fr:Lait de chèvre entier UHT
+bg:пълномаслено козе мляко УХТ
+ca:llet sencer de cabra UHT
+de:Ziegenvollmilch UHT
+es:leche entera de cabra UHT
+fi:vuohen täysmaito UHT
+hu:teljes kecsketej UHT
+nl:volle geitenmelk UHT
 ciqual_food_code:en:19200
 ciqual_food_name:en:Goat milk, whole, UHT
 ciqual_food_name:fr:Lait de chèvre, entier, UHT
@@ -3536,6 +3556,13 @@ ciqual_food_name:fr:Lait de chèvre, entier, UHT
 <en:semi-skimmed goat milk
 en:semi-skimmed pasteurized goat milk UHT
 fr:lait de chèvre demi-écrémé UHT
+bg:полуобезмаслено пастьоризирано козе мляко УХТ
+ca:llet de cabra semidesnatada pasteuritzada UHT
+de:teilentrahmte pasteurisierte Ziegenmilch UHT
+es:leche de cabra semidesnatada pasteurizada UHT
+fi:kevyesti pastöroitu puolirasvainen vuohenmaito UHT
+hu:zsírszegény pasztőrözött kecsketej UHT
+nl:halfvolle gepasteuriseerde geitenmelk UHT
 ciqual_food_code:en:19201
 ciqual_food_name:en:Goat milk, half skimmed, UHT pasteurized
 ciqual_food_name:fr:Lait de chèvre, demi-écrémé, UHT
@@ -3599,6 +3626,10 @@ description:en:Sheep's milk (or ewes' milk) is the milk of domestic sheep.
 <en:goat milk
 en:Sheeps or goat milk
 jv:susu wedhus
+fr:lait de brebis ou de chèvre
+es:leche de oveja o de cabra
+de:Schaf- oder Ziegenmilch
+nl:schapen- of geitenmelk
 
 <en:sheeps milk
 en:whole sheep milk
@@ -3616,6 +3647,14 @@ ciqual_food_name:fr:Lait de brebis, entier
 <en:skimmed milk
 en:skimmed sheeps milk, skimmed sheep milk, semi-skimmed sheeps milk
 fr:lait de brebis partiellement écrémé
+ar:حليب الغنم قليل الدسم
+bg:нискомаслено овче мляко
+ca:llet d'ovella desnatada, llet d'ovella descremada
+de:fettarme Schafmilch, fettreduzierte Schafmilch
+es:leche de oveja desnatada, leche de oveja semidesnatada
+fi:rasvaton lampaanmaito, vähärasvainen lampaanmaito
+hu:zsírszegény juhtej
+nl:mager schapenmelk
 
 #processing:en: en:raw
 #<en:sheeps milk
@@ -3650,6 +3689,8 @@ fr:lait de brebis partiellement écrémé
 fr:lait de brebis thermisé
 ca:llet d'ovella termitzada, llet termitzada d'ovella
 es:leche de oveja termizada, leche termizada de oveja
+it:latte di pecora termizzato
+nl:thermische schapenmelk
 
 
 # <en:aggregate
@@ -7074,6 +7115,8 @@ nl:verse kaas
 pt:queijo branco
 sl:sveži sir 
 sr:sveži sir
+ciqual_food_code:19501
+ciqual_food_name:en:Quark, plain or fruit (average)
 # en-ca:Fromage blanc
 # en-gb:Fromage blanc
 wikidata:en:Q3323634
@@ -29364,6 +29407,10 @@ wikipedia:en:https://en.wikipedia.org/wiki/Nib_sugar
 
 <en:sugar
 fr:sucre blond
+en:blonde sugar
+es:azúcar rubio
+it:zucchero biondo
+nl:blonde suiker
 
 # description:en:Granulated, familiar as table sugar, with a grain size about 0.5-0.6 mm across.
 # comment:en:"granulated sugar" is generally white sugar, but other sugars can be in granulated form too.
@@ -29418,6 +29465,11 @@ vegetarian:en:yes
 <en:sugarcane
 en:sugarcane juice, cane juice
 fr:jus de canne à sucre, jus de canne
+bg:сок от захарна тръстика
+es:jugo de caña de azúcar
+de:Zuckerrohrsaft
+it:succo di canna da zucchero
+nl:suikerrietsap
 
 <en:sugar
 nl:Arenga bossuiker
@@ -29613,14 +29665,16 @@ wikidata:en:Q10357458
 wikipedia:en:https://en.wikipedia.org/wiki/Panela
 
 <en:sugar
-en:beet sugar
+en:beet sugar, beetroot sugar
 de:Rübenzucker
+es:azucar de remolacha
 fi:juurikassokeri
 fr:sucre de betterave
 hr:šećer šećerne repe
 hu:répacukor
 it:Zucchero di barbabietola
-nl:suikerbietsuiker
+nl:suikerbietsuiker, bietensuiker
+pt:açúcar de beterraba
 carbon_footprint_fr_foodges_ingredient:fr:Sucre de betterave
 carbon_footprint_fr_foodges_value:fr:0.7
 
@@ -48636,6 +48690,9 @@ zh:香芹
 # xmf:მაკინდოლი
 # yue:番茜
 wikidata:en:q65522500
+ciqual_proxy_food_code:en:11014
+ciqual_proxy_food_name:en:Parsley, fresh
+ciqual_proxy_food_name:fr:Persil, frais
 # plant: wikidata:en:q25284
 wikipedia:en:https://en.wikipedia.org/wiki/parsley
 # en:22363 in 51 @2021-09-16
@@ -53728,10 +53785,6 @@ eurocode_2_group_3:en: 8.38.55
 #pl:burak w proszku, proszek z buraka
 # fr:poudre-de-betterave has 55 products in 1 language @2018-10-22
 
-<en:beetroot
-en:beetroot sugar
-es:azucar de remolacha
-
 <en:beetroot
 en:beetroot syrup
 de:Rote-Beete-Sirup
@@ -53768,6 +53821,20 @@ sk:koncentrát šťavy z cvikly
 
 <en:beetroot
 fr:concentré de betterave
+bg:концентрат от цвекло
+cs:koncentrát z červené řepy
+de:Rote-Bete-Konzentrat
+es:concentrado de remolacha
+fi:punajuuritiiviste
+it:concentrato di barbabietola
+ja:ビーツ濃縮液
+lt:raudonųjų burokėlių koncentratas
+nl:bietensapconcentraat
+pl:koncentrat z buraka
+pt:concentrado de beterraba
+ru:концентрат свеклы
+sk:koncentrát z cvikly
+sv:rödbetskoncentrat
 
 <en:beetroot
 # <en:fibre
@@ -54471,6 +54538,10 @@ fr:salade mélangée verte et rouge
 # <en:compound
 <en:salad
 fr:salade composée
+en:mixed salad
+de:gemischter Salat
+es:ensalada mixta
+it:insalata mista
 # ingredient/fr:salade-composée has 97 products in french @2019-04-16
 # usage:fr: Salade composée 88% : Scarole, carotte râpées, maïs (maïs, sucre, sel), emmental 12%, dés de jambon cuit supérieur 12% (jambon de porc 10,9%, sel, sirop de glucose, dextrose, arômes naturels, antioxydant : ascorbate de sodium, conservateur: nitrite de sodium), frisée, chicorée rouge.
 
@@ -61378,6 +61449,8 @@ zh:芝麻
 # zh-tw:胡麻
 allergens:en: en:sesame seeds
 wikidata:en:Q2763698
+ciqual_food_code:15010
+ciqual_food_name:en:Sesame seed
 # ingredient/fr:sésame has 1745 products @2018-10-05
 
 <en:sesame
@@ -62441,6 +62514,9 @@ uk:куркума
 ur:ہلدی
 vi:nghệ
 zh:薑黃
+ciqual_proxy_food_code:en:11089
+ciqual_proxy_food_name:en:Turmeric, powder
+ciqual_proxy_food_name:fr:Curcuma, poudre
 # ace:kunyèt
 # azb:kurkuma
 # bcl:kalawag
@@ -64021,6 +64097,8 @@ wo:anet
 zh:蒔蘿
 description:en:dill (anethum graveolens) is an annual herb in the celery family apiaceae. dill is widely grown in eurasia where its leaves and seeds are used as a herb or spice for flavouring food.
 comment:en:the wikidata page mixes the common name and latin name
+ciqual_food_code:11093
+ciqual_food_name:en:Dill, fresh
 # aeb-arab:شبت
 # aeb-latn:ch'ebt
 # arz:شبت
@@ -65804,6 +65882,9 @@ sl:origano
 uk:орегано
 xx:oregano
 wikipedia:en:Q92581152
+ciqual_proxy_food_code:en:11035
+ciqual_proxy_food_name:en:Oregano, dried
+ciqual_proxy_food_name:fr:Origan, séché
 #plant: wikipedia:en:Q134283
 # plante:https://fr.wikipedia.org/wiki/Origanum_vulgare
 # 9423 in 29 @2021-09-17
@@ -72694,9 +72775,13 @@ sv:torskfilé, filér av torsk
 # ingredient/fr:filet-de-cabillaud has 32 products in french @2018-12-29
 
 <en:cod
+en:smoked cod roe
 fr:oeufs de cabillaud fumés
+es:huevas de bacalao ahumadas
+de:geräucherter Kabeljaukaviar
+it:uova di merluzzo affumicate
 
-<fr:oeufs de cabillaud fumés
+<en:smoked cod roe
 fr:oeufs de cabillaud salés et fumés, œufs de cabillaud fumés et salés, oeufs de cabillaud fumés et salés
 # ingredient/fr:œufs-de-cabillaud-fumés-et-salés has 30 products in french @2018-12-29
 
@@ -73833,6 +73918,8 @@ en:pangas fillet
 de:Pangasiusfilets
 fr:filets de pangas
 it:filetti di pangasio
+es:filetes de pangas
+nl:pangasiusfilets
 
 #description:en:Trisopterus luscus (bib, pout whiting, pout or most commonly pouting) is a seafish belonging to the cod family (Gadidae).
 
@@ -74055,6 +74142,10 @@ de:Roher Lachs
 fr:saumon cru
 nl:rauwe zalm
 pt:Salmão cru
+es:salmon crudo
+it:salmone crudo
+pl:surowy łosoś
+ru:сырая лосось
 # ingredient/raw-salmon has 100 products in 3 languages @2021-08-15
 ciqual_food_code:en:26036
 ciqual_food_name:en:Salmon, raw, farmed
@@ -74064,6 +74155,9 @@ ciqual_food_name:fr:Saumon, cru, élevage
 <en:raw salmon
 en:raw farmed Salmon
 fr:Saumon cru d'élevage
+es:Salmón crudo de piscifactoría
+de:Roher gezüchteter Lachs
+it:Salmone crudo di allevamento
 ciqual_food_code:en:26036
 ciqual_food_name:en:Salmon, raw, farmed
 ciqual_food_name:fr:Saumon, cru, élevage
@@ -74072,6 +74166,9 @@ ciqual_food_name:fr:Saumon, cru, élevage
 <en:wild salmon
 en:raw wild Salmon
 fr:Saumon cru sauvage
+es:Salmón crudo salvaje
+de:Roher wilder Lachs
+it:Salmone crudo selvaggio
 ciqual_food_code:en:26161
 ciqual_food_name:en:Salmon, raw, wild
 ciqual_food_name:fr:Saumon, cru, sauvage
@@ -74137,11 +74234,19 @@ ciqual_food_name:en:Salmon, smoked
 
 <en:smoked salmon
 fr:chutes de saumon fumé
+en:smoked salmon trimmings
+de:geräucherte Lachsschnitte
+es:recortes de salmón ahumado
+it:scarti di salmone affumicato
+nl:gerookte zalmresten
 
 <en:salmon
 en:scottish salmon
 de:schottischer Lachs
 fr:saumon d'écosse
+es:salmon escocés
+it:salmone scozzese
+nl:Schotse zalm
 
 #description:en:The Atlantic salmon (Salmo salar) is a species of ray-finned fish in the family Salmonidae.
 
@@ -74226,6 +74331,9 @@ en:smoked atlantic salmon
 bg:пушена атлантическа сьомга
 de:geräucherter Atlantik-Lachs, geräucherter Atlantiklachs
 fr:saumon Atlantique fumé
+es:salmón atlántico ahumado
+it:salmone atlantico affumicato
+pt:salmão atlântico defumado
 
 
 <en:salmon fillet
@@ -74450,15 +74558,16 @@ ciqual_food_name:fr:Sardine, crue
 # 1553 in 20 @2021-09-15
 
 #processing:en: en:fillet
-#<en:sardine
-#en:sardine fillets
-#de:Sardinenfilets
-#fi:sardiinifilee
-#fr:filets de sardines
-#it:filetti di sardine
-#nl:sardinefilets
-#sk:filety sardiniek
+<en:sardine
+en:sardine fillets
+de:Sardinenfilets
+fi:sardiinifilee
+fr:filets de sardines
+it:filetti di sardine
+nl:sardinefilets
+sk:filety sardiniek
 # 142 in 5 languages @2021-09-21
+# We disabled it because we can catch it in processing, but as a result, it does not inherit the CIQUAL property of sardines
 
 #processing:en: en:pre-cooked
 #<en:sardine fillets
@@ -74533,6 +74642,9 @@ wikipedia:en:https://en.wikipedia.org/wiki/Scorpaenidae
 <en:scorpion fish
 en:Steamed spiny scorpionfish
 fr:Rascasse cuite à la vapeur
+es:Rascacio al vapor
+de:Gedämpfter Drachenkopf
+it:Scorfano spinoso al vapore
 ciqual_food_code:en:26148
 ciqual_food_name:en:Spiny scorpionfish, steamed
 ciqual_food_name:fr:Rascasse, cuite à la vapeur
@@ -74564,6 +74676,10 @@ zh:龍脷
 <en:fish
 en:Tonguesole
 fr:sole tropicale, sole langue
+en:tropical sole, tongue sole
+es:lenguado tropical, lenguado lengua
+de:tropische Seezunge, Zungenzunge
+it:sole tropicale, sogliola lingua
 la:Cynoglossus spp
 ciqual_food_code:en:26200
 ciqual_food_name:en:Tonguesole, raw
diff --git a/tests/unit/expected_test_results/ingredients/fi-do-not-match-myanmar.json b/tests/unit/expected_test_results/ingredients/fi-do-not-match-myanmar.json
index 845ed584f6b38..8e1048a6754da 100644
--- a/tests/unit/expected_test_results/ingredients/fi-do-not-match-myanmar.json
+++ b/tests/unit/expected_test_results/ingredients/fi-do-not-match-myanmar.json
@@ -4,6 +4,7 @@
          "id" : "en:spice",
          "ingredients" : [
             {
+               "ciqual_proxy_food_code" : "11089",
                "id" : "en:turmeric",
                "percent_estimate" : 58.3333333333333,
                "percent_max" : 100,
@@ -132,10 +133,9 @@
    "ingredients_with_unspecified_percent_sum" : 100,
    "ingredients_without_ciqual_codes" : [
       "en:coriander",
-      "en:spice",
-      "en:turmeric"
+      "en:spice"
    ],
-   "ingredients_without_ciqual_codes_n" : 3,
+   "ingredients_without_ciqual_codes_n" : 2,
    "known_ingredients_n" : 15,
    "lc" : "fi",
    "nutriments" : {