feat: object export (#812)

ecobalyse-private: object-export-ccomb ## 🔧 Problem We copy/paste impacts for object domain -> no traceability ## 🍰 Solution Compute and export impacts for object like we do for food ## 🏝️ How to test in ecobalyse/data execute `make export_food` check that there isn't any diff ecobalyse-private: object-export-ccomb --------- Co-authored-by: paulboosz <[email protected]>
MTES-MCT · Nov 5, 2024 · 4754202 · 4754202
1 parent b08a789
commit 4754202
Show file tree

Hide file tree

Showing 17 changed files with 1,275 additions and 1,302 deletions.
diff --git a/compute-aggregated.js b/compute-aggregated.js
diff --git a/data/Makefile b/data/Makefile
@@ -16,21 +16,24 @@ else \
 endef
 
 all: import export
-import : image import_food import_ecoinvent import_method sync_datapackages
-export: export_food export_textile format
+import : image import_food import_ecoinvent import_method create_activities sync_datapackages
+export: export_food export_textile export_object format
 
 image:
 	docker build -t $(NAME) docker
 
 import_food:
-	@$(call DOCKER,python3 import_food.py --recreate-activities)
+	@$(call DOCKER,python3 import_food.py)
 
 import_method:
 	@$(call DOCKER,python3 import_method.py)
 
 import_ecoinvent:
 	@$(call DOCKER,python3 import_ecoinvent.py)
 
+create_activities:
+	@$(call DOCKER,python3 create_activities.py)
+
 sync_datapackages:
 	@$(call DOCKER,python3 common/sync_datapackages.py)
 
@@ -46,6 +49,9 @@ export_food:
 export_textile:
 	@$(call DOCKER,bash -c "python3 textile/export.py")
 
+export_object:
+	@$(call DOCKER,bash -c "python3 object/export.py")
+
 compare_food:
 	@$(call DOCKER,bash -c "python3 food/export.py compare")
 

diff --git a/data/common/__init__.py b/data/common/__init__.py
@@ -1,5 +1,4 @@
 # Please only pure functions here
-
 from copy import deepcopy
 
 from frozendict import frozendict

diff --git a/data/common/export.py b/data/common/export.py
@@ -1,6 +1,6 @@
 import functools
 import json
-import logging
+import sys
 import urllib.parse
 from os.path import dirname
 
@@ -12,6 +12,7 @@
 import requests
 from bw2io.utils import activity_hash
 from frozendict import frozendict
+from loguru import logger
 
 from . import (
     bytrigram,
@@ -22,14 +23,19 @@
 )
 from .impacts import main_method
 
-logging.basicConfig(level=logging.ERROR)
+# Configure logger
+logger.remove()  # Remove default handler
+logger.add(sys.stderr, format="{time} {level} {message}", level="INFO")
 
 PROJECT_ROOT_DIR = dirname(dirname(dirname(__file__)))
 COMPARED_IMPACTS_FILE = "compared_impacts.csv"
 
 with open(f"{PROJECT_ROOT_DIR}/public/data/impacts.json") as f:
     IMPACTS_JSON = json.load(f)
 
+PROJECT_ROOT_DIR = dirname(dirname(dirname(__file__)))
+COMPARED_IMPACTS_FILE = "compared_impacts.csv"
+
 
 def check_ids(ingredients):
     # Check the id is lowercase and does not contain space
@@ -129,7 +135,7 @@ def create_activity(dbname, new_activity_name, base_activity=None):
         new_activity["code"] = code
     new_activity["Process identifier"] = code
     new_activity.save()
-    logging.info(f"Created activity {new_activity}")
+    logger.info(f"Created activity {new_activity}")
     return new_activity
 
 
@@ -142,16 +148,16 @@ def delete_exchange(activity, activity_to_delete, amount=False):
                 and exchange["amount"] == amount
             ):
                 exchange.delete()
-                logging.info(f"Deleted {exchange}")
+                logger.info(f"Deleted {exchange}")
                 return
 
     else:
         for exchange in activity.exchanges():
             if exchange.input["name"] == activity_to_delete["name"]:
                 exchange.delete()
-                logging.info(f"Deleted {exchange}")
+                logger.info(f"Deleted {exchange}")
                 return
-    logging.error(f"Did not find exchange {activity_to_delete}. No exchange deleted")
+    logger.error(f"Did not find exchange {activity_to_delete}. No exchange deleted")
 
 
 def new_exchange(activity, new_activity, new_amount=None, activity_to_copy_from=None):
@@ -165,7 +171,7 @@ def new_exchange(activity, new_activity, new_amount=None, activity_to_copy_from=
                 new_amount = exchange["amount"]
                 break
         else:
-            logging.error(
+            logger.error(
                 f"Exchange to duplicate from :{activity_to_copy_from} not found. No exchange added"
             )
             return
@@ -179,7 +185,7 @@ def new_exchange(activity, new_activity, new_amount=None, activity_to_copy_from=
         comment="added by Ecobalyse",
     )
     new_exchange.save()
-    logging.info(f"Exchange {new_activity} added with amount: {new_amount}")
+    logger.info(f"Exchange {new_activity} added with amount: {new_amount}")
 
 
 def compute_impacts(frozen_processes, default_db, impacts_py):
@@ -205,15 +211,17 @@ def compute_impacts(frozen_processes, default_db, impacts_py):
     }
     """
     processes = dict(frozen_processes)
-    print("Computing impacts:")
+    logger.info("Computing impacts:")
     for index, (_, process) in enumerate(processes.items()):
         progress_bar(index, len(processes))
         # Don't compute impacts if its a hardcoded activity
-        if process["impacts"]:
-            print(f"This process has hardcoded impacts: {process['displayName']}")
+        if process.get("impacts"):
+            logger.info(f"This process has hardcoded impacts: {process['displayName']}")
             continue
         # simapro
-        activity = cached_search(process.get("source", default_db), process["search"])
+        activity = cached_search(
+            process.get("source", default_db), process.get("search", process["name"])
+        )
         if not activity:
             raise Exception(f"This process was not found in brightway: {process}")
 
@@ -229,7 +237,7 @@ def compute_impacts(frozen_processes, default_db, impacts_py):
         if isinstance(results, dict) and results:
             # simapro succeeded
             process["impacts"] = results
-            print(f"got impacts from simapro for: {process['name']}")
+            logger.info(f"got impacts from simapro for: {process['name']}")
         else:
             # simapro failed (unexisting Ecobalyse project or some other reason)
             # brightway
@@ -252,7 +260,7 @@ def compute_impacts(frozen_processes, default_db, impacts_py):
 def compare_impacts(frozen_processes, default_db, impacts_py, impacts_json):
     """This is compute_impacts slightly modified to store impacts from both bw and sp"""
     processes = dict(frozen_processes)
-    print("Computing impacts:")
+    logger.info("Computing impacts:")
     for index, (key, process) in enumerate(processes.items()):
         progress_bar(index, len(processes))
         # simapro
@@ -261,10 +269,10 @@ def compare_impacts(frozen_processes, default_db, impacts_py, impacts_json):
             process.get("search", process["name"]),
         )
         if not activity:
-            print(f"{process['name']} does not exist in brightway")
+            logger.info(f"{process['name']} does not exist in brightway")
             continue
         results = compute_simapro_impacts(activity, main_method, impacts_py)
-        print(f"got impacts from SimaPro for: {process['name']}")
+        logger.info(f"got impacts from SimaPro for: {process['name']}")
 
         # WARNING assume remote is in m3 or MJ (couldn't find unit from COM intf)
         if process["unit"] == "kWh" and isinstance(results, dict):
@@ -278,7 +286,7 @@ def compare_impacts(frozen_processes, default_db, impacts_py, impacts_json):
         process["brightway_impacts"] = compute_brightway_impacts(
             activity, main_method, impacts_py
         )
-        print(f"got impacts from Brightway for: {process['name']}")
+        logger.info(f"got impacts from Brightway for: {process['name']}")
 
         # compute subimpacts
         process["simapro_impacts"] = with_subimpacts(process["simapro_impacts"])
@@ -350,11 +358,11 @@ def csv_export_impact_comparison(compared_impacts, folder):
 
 
 def export_json(json_data, filename):
-    print(f"Exporting {filename}")
+    logger.info(f"Exporting {filename}")
     with open(filename, "w", encoding="utf-8") as file:
         json.dump(json_data, file, indent=2, ensure_ascii=False)
         file.write("\n")  # Add a newline at the end of the file
-    print(f"\nExported {len(json_data)} elements to {filename}")
+    logger.info(f"\nExported {len(json_data)} elements to {filename}")
 
 
 def load_json(filename):
@@ -380,13 +388,11 @@ def compute_simapro_impacts(activity, method, impacts_py):
     strprocess = urllib.parse.quote(activity["name"], encoding=None, errors=None)
     project = urllib.parse.quote(spproject(activity), encoding=None, errors=None)
     method = urllib.parse.quote(main_method, encoding=None, errors=None)
+    api_request = f"http://simapro.ecobalyse.fr:8000/impact?process={strprocess}&project={project}&method={method}"
+    logger.debug(f"SimaPro API request: {api_request}")
     return bytrigram(
         impacts_py,
-        json.loads(
-            requests.get(
-                f"http://simapro.ecobalyse.fr:8000/impact?process={strprocess}&project={project}&method={method}"
-            ).content
-        ),
+        json.loads(requests.get(api_request).content),
     )
 
 
@@ -398,4 +404,5 @@ def compute_brightway_impacts(activity, method, impacts_py):
         lca.switch_method(method)
         lca.lcia()
         results[key] = float("{:.10g}".format(lca.score))
+        logger.debug(f"{activity}  {key}: {lca.score}")
     return results
diff --git a/data/create_activities.py b/data/create_activities.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+import os
+
+import bw2data
+from common.import_ import add_created_activities
+
+if __name__ == "__main__":
+    """Add additional processes"""
+
+    if "Ecobalyse" in bw2data.databases:
+        del bw2data.databases["Ecobalyse"]
+
+    if (db := "Ecobalyse") not in bw2data.databases:
+        for vertical in ("object", "food", "textile"):
+            file = f"{vertical}/activities_to_create.json"
+            if os.path.exists(file):
+                add_created_activities(db, file)
+    else:
+        print(f"{db} already imported")
diff --git a/data/docker/requirements.txt b/data/docker/requirements.txt
@@ -22,3 +22,4 @@ pandas>=2.0, <2.1
 scipy>=1.11, <1.12
 uvicorn
 xlrd>=2.0, <2.1
+loguru>=0.7.2, <0.8
diff --git a/data/object/__init__.py b/data/object/__init__.py
diff --git a/data/object/activities.json b/data/object/activities.json
@@ -0,0 +1,20 @@
+[
+  {
+    "id": "07e9e916-e02b-45e2-a298-2b5084de6242",
+    "name": "sawnwood, board, hardwood, dried (u=10%), planed//[Europe without Switzerland] market for sawnwood, board, hardwood, dried (u=10%), planed",
+    "displayName": "Planche (bois de feuillus)",
+    "density": 600,
+    "unit": "m3",
+    "source": "Ecoinvent 3.9.1",
+    "comment": ""
+  },
+  {
+    "id": "3295b2a5-328a-4c00-b046-e2ddeb0da823",
+    "name": "polypropylene product/RER constructed by Ecobalyse",
+    "displayName": "Composant en plastique (PP)",
+    "density": 900,
+    "unit": "kg",
+    "source": "Ecobalyse",
+    "comment": "modélisation d'un composant générique (voir documentation)"
+  }
+]
diff --git a/data/object/activities_to_create.json b/data/object/activities_to_create.json
@@ -0,0 +1,13 @@
+[
+  {
+    "alias": "plastic-product-pp",
+    "search_in": "Ecoinvent 3.9.1",
+    "search": "polypropylene product/RER",
+    "suffix": "constructed by Ecobalyse",
+    "subactivities": [],
+    "add": {
+      "polypropylene, granulate//[RER] polypropylene production, granulate": 1.01,
+      "injection moulding//[RER] injection moulding": 1.01
+    }
+  }
+]