Skip to content

Commit

Permalink
fix persisting nan values in EJR #632
Browse files Browse the repository at this point in the history 
EJR update job_id='j-24011972cfb04efcb15e28d0b230a23b' data={'costs': 5.0, 'usage': {'cpu': {'value': 4033, 'unit': 'cpu-seconds'}, 'memory': {'value': 9359879, 'unit': 'mb-seconds'}, 'input_pixel': {'value': 0.0625, 'unit': 'mega-pixel'}}, 'results_metadata': {'geometry': {'type': 'Polygon', 'coordinates': [[[4.825919, 51.259766], [4.825919, 51.307638], [4.859629, 51.307638], [4.859629, 51.259766], [4.825919, 51.259766]]]}, 'bbox': [4.825919, 51.259766, 4.859629, 51.307638], 'area': {'value': 12526090.219329834, 'unit': 'square meter'}, 'start_datetime': '2017-11-01T00:00:00Z', 'end_datetime': '2017-11-01T00:00:00Z', 'links': [{'href': 'urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'rel': 'derived_from', 'title': 'Derived from urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'type': 'application/json'}, {'href': 'urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'rel': 'derived_from', 'title': 'Derived from urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'type': 'application/json'}, {'href': 'urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'rel': 'derived_from', 'title': 'Derived from urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'type': 'application/json'}, {'href': 'urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'rel': 'derived_from', 'title': 'Derived from urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'type': 'application/json'}, {'href': 'urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'rel': 'derived_from', 'title': 'Derived from urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'type': 'application/json'}, {'href': 'urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'rel': 'derived_from', 'title': 'Derived from urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'type': 'application/json'}, {'href': 'urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'rel': 'derived_from', 'title': 'Derived from urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'type': 'application/json'}, {'href': 'urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'rel': 'derived_from', 'title': 'Derived from urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'type': 'application/json'}, {'href': 'urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'rel': 'derived_from', 'title': 'Derived from urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'type': 'application/json'}, {'href': 'urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'rel': 'derived_from', 'title': 'Derived from urn:eop:VITO:PROBAV_S10_TOC_333M_COG_V2:PROBAV_S10_TOC_X18Y02_20171101_333M_V201', 'type': 'application/json'}], 'proj:bbox': [4.8257395, 51.2594548, 4.8614451, 51.3088932], 'proj:shape': [13, 36], 'assets': {'openEO.tif': {'href': '/data/projects/OpenEO/j-24011972cfb04efcb15e28d0b230a23b/openEO.tif', 'type': 'image/tiff; application=geotiff', 'roles': ['data'], 'bands': [], 'nodata': nan, 'datetime': None, 'raster:bands': [{'name': '1', 'statistics': {'minimum': 3.0, 'maximum': 5.0, 'mean': 4.2350427350427, 'stddev': 0.97198503728382, 'valid_percent': 100.0}}]}}, 'epsg': 4326, 'instruments': ['PROBA-V'], 'processing:facility': 'VITO - SPARK', 'processing:software': 'openeo-geotrellis-0.23.0a1', 'unique_process_ids': ['reduce_dimension', 'load_ml_model', 'save_result', 'load_collection', 'predict_random_forest', 'mean'], 'providers': [{'name': 'VITO', 'description': 'This data was processed on an openEO backend maintained by VITO.', 'roles': ['processor'], 'processing:facility': 'openEO Geotrellis backend', 'processing:software': {'Geotrellis backend': '0.23.0a1'}, 'processing:expression': [{'format': 'openeo', 'expression': {'loadmlmodel1': {'process_id': 'load_ml_model', 'arguments': {'id': 'j-240119d8e2234977beeb86f307291756'}}, 'loadcollection1': {'process_id': 'load_collection', 'arguments': {'bands': ['NDVI'], 'id': 'PROBAV_L3_S10_TOC_333M', 'spatial_extent': {'west': 4.825919, 'east': 4.859629, 'south': 51.259766, 'north': 51.307638}, 'temporal_extent': ['2017-11-01', '2017-11-01']}}, 'reducedimension1': {'process_id': 'reduce_dimension', 'arguments': {'data': {'from_node': 'loadcollection1'}, 'dimension': 't', 'reducer': {'process_graph': {'mean1': {'process_id': 'mean', 'arguments': {'data': {'from_parameter': 'data'}}, 'result': True}}}}}, 'reducedimension2': {'process_id': 'reduce_dimension', 'arguments': {'context': {'from_node': 'loadmlmodel1'}, 'data': {'from_node': 'reducedimension1'}, 'dimension': 'bands', 'reducer': {'process_graph': {'predictrandomforest1': {'process_id': 'predict_random_forest', 'arguments': {'data': {'from_parameter': 'data'}, 'model': {'from_parameter': 'context'}}, 'result': True}}}}}, 'saveresult1': {'process_id': 'save_result', 'arguments': {'data': {'from_node': 'reducedimension2'}, 'format': 'GTiff', 'options': {}}, 'result': True}}}]}], 'usage': {'input_pixel': {'value': 0.0625, 'unit': 'mega-pixel'}}}}

Traceback (most recent call last):
  File "/opt/venv/lib64/python3.8/site-packages/requests/models.py", line 511, in prepare_body
    body = complexjson.dumps(json, allow_nan=False)
  File "/usr/lib64/python3.8/json/__init__.py", line 234, in dumps
    return cls(
  File "/usr/lib64/python3.8/json/encoder.py", line 199, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "/usr/lib64/python3.8/json/encoder.py", line 257, in iterencode
    return _iterencode(o, 0)
ValueError: Out of range float values are not JSON compliant

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/job_tracker_v2.py", line 430, in update_statuses
    self._sync_job_status(
  File "/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/job_tracker_v2.py", line 553, in _sync_job_status
    double_job_registry.set_results_metadata(job_id, user_id, costs=job_costs, usage=dict(total_usage),
  File "/opt/venv/lib64/python3.8/site-packages/openeogeotrellis/job_registry.py", line 964, in set_results_metadata
    self.elastic_job_registry.set_results_metadata(job_id=job_id, costs=costs, usage=usage,
  File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/jobregistry.py", line 553, in set_results_metadata
    return self._update(job_id=job_id, data={
  File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/jobregistry.py", line 460, in _update
    return self._do_request("PATCH", f"/jobs/{job_id}", json=data)
  File "/opt/venv/lib64/python3.8/site-packages/openeo_driver/jobregistry.py", line 297, in _do_request
    response = self._session.request(
  File "/opt/venv/lib64/python3.8/site-packages/requests/sessions.py", line 575, in request
    prep = self.prepare_request(req)
  File "/opt/venv/lib64/python3.8/site-packages/requests/sessions.py", line 486, in prepare_request
    p.prepare(
  File "/opt/venv/lib64/python3.8/site-packages/requests/models.py", line 371, in prepare
    self.prepare_body(data, files, json)
  File "/opt/venv/lib64/python3.8/site-packages/requests/models.py", line 513, in prepare_body
    raise InvalidJSONError(ve, request=self)
requests.exceptions.InvalidJSONError: Out of range float values are not JSON compliant
  • Loading branch information
@bossie
bossie committed Jan 19, 2024
1 parent d75bd14 commit 38a2600
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 4 deletions.
21 changes: 19 additions & 2 deletions openeogeotrellis/job_tracker_v2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import datetime as dt
import logging
from decimal import Decimal
from math import isfinite
from pathlib import Path
from typing import Any, List, NamedTuple, Optional, Union

Expand Down Expand Up @@ -550,8 +551,9 @@ def _sync_job_status(
job_costs = None

total_usage = dict_merge_recursive(job_metadata.usage.to_dict(), result_metadata.get("usage", {}))
double_job_registry.set_results_metadata(job_id, user_id, costs=job_costs, usage=dict(total_usage),
results_metadata=result_metadata)
double_job_registry.set_results_metadata(job_id, user_id, costs=job_costs,
usage=self._to_jsonable(dict(total_usage)),
results_metadata=self._to_jsonable(result_metadata))

datetime_formatter = Rfc3339(propagate_none=True)

Expand All @@ -563,6 +565,21 @@ def _sync_job_status(
finished=datetime_formatter.datetime(job_metadata.finish_time),
)

@staticmethod
def _to_jsonable_float(x: float) -> Union[float, str]:
return x if isfinite(x) else str(x)

@staticmethod
def _to_jsonable(x):
if isinstance(x, float):
return JobTracker._to_jsonable_float(x)
if isinstance(x, dict):
return {JobTracker._to_jsonable(key): JobTracker._to_jsonable(value) for key, value in x.items()}
elif isinstance(x, list):
return [JobTracker._to_jsonable(elem) for elem in x]

return x


class CliApp:
def main(self, *, args: Optional[List[str]] = None):
Expand Down
10 changes: 8 additions & 2 deletions tests/test_job_tracker_v2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1574,7 +1574,9 @@ def test_k8s_no_zookeeper(self, k8s_mock, prometheus_mock, job_costs_calculator,
data={
"foo": "bar",
"usage": {"input_pixel": {"unit": "mega-pixel", "value": 1.125},
"sentinelhub": {"unit": "sentinelhub_processing_unit", "value": 1.25},}
"sentinelhub": {"unit": "sentinelhub_processing_unit", "value": 1.25},
},
"b0rked": [24, ["a", {"b": [float('nan'), None]}], {"c": [float('inf'), 1.23]}],
},
)
job_tracker.update_statuses()
Expand All @@ -1597,11 +1599,15 @@ def test_k8s_no_zookeeper(self, k8s_mock, prometheus_mock, job_costs_calculator,
"results_metadata": {
"foo": "bar",
"usage": {"input_pixel": {"unit": "mega-pixel", "value": 1.125},
"sentinelhub": {"unit": "sentinelhub_processing_unit", "value": 1.25},}
"sentinelhub": {"unit": "sentinelhub_processing_unit", "value": 1.25},
},
"b0rked": [24, ["a", {"b": ['nan', None]}], {"c": ['inf', 1.23]}],
},
}
)

json.dumps(elastic_job_registry.db[job_id], allow_nan=False)

assert caplog.record_tuples == []


Expand Down

0 comments on commit 38a2600

Please sign in to comment.