From 08fff94d5b673833f4c5049ac1ba923738488996 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Wed, 29 Apr 2026 16:10:20 +0200 Subject: [PATCH 1/6] provide pg/tracer to log added value from https://github.com/Open-EO/openeo-geopyspark-driver/issues/1436 --- openeo_driver/backend.py | 5 ++++- openeo_driver/processgraph/registry.py | 10 +++++----- openeo_driver/views.py | 19 ++++++++++++++++--- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/openeo_driver/backend.py b/openeo_driver/backend.py index f6c805eb..0cc49b2c 100644 --- a/openeo_driver/backend.py +++ b/openeo_driver/backend.py @@ -19,6 +19,7 @@ from typing import List, Union, NamedTuple, Dict, Optional, Callable, Iterable, Container, Any, Tuple import flask +from openeo_driver.processgraph.definitions import ProcessGraphFlatDict import openeo_driver.util.view_helpers from openeo.utils.version import ComparableVersion @@ -786,7 +787,7 @@ class Processing(MicroService): def get_process_registry(self, api_version: Union[str, ComparableVersion]) -> ProcessRegistry: raise NotImplementedError - def evaluate(self, process_graph: dict, env: EvalEnv = None): + def evaluate(self, process_graph: dict, env: EvalEnv = None, do_dry_run: Union[bool, DryRunDataTracer] = True): """Evaluate given process graph (flat dict format).""" raise NotImplementedError @@ -1060,9 +1061,11 @@ def request_costs( self, *, user: User, + process_graph: ProcessGraphFlatDict, job_options: Union[dict, None] = None, request_id: str, success: bool, + tracer: DryRunDataTracer, ) -> Optional[float]: """ Report resource usage of (current) synchronous processing request and get associated cost. diff --git a/openeo_driver/processgraph/registry.py b/openeo_driver/processgraph/registry.py index 24819bea..556a76ba 100644 --- a/openeo_driver/processgraph/registry.py +++ b/openeo_driver/processgraph/registry.py @@ -13,6 +13,7 @@ from openeo.utils.version import ComparableVersion from openeo_driver.backend import OpenEoBackendImplementation, Processing +from openeo_driver.dry_run import DryRunDataTracer from openeo_driver.errors import OpenEOApiException from openeo_driver.processes import DEFAULT_NAMESPACE, ProcessArgs, ProcessRegistry, ProcessSpec from openeo_driver.specs import SPECS_ROOT @@ -238,7 +239,7 @@ def get_basic_env(self, api_version: str = OPENEO_API_VERSION_DEFAULT) -> EvalEn } ) - def evaluate(self, process_graph: dict, env: EvalEnv = None): + def evaluate(self, process_graph: dict, env: EvalEnv = None, do_dry_run: Union[bool, DryRunDataTracer] = True): from openeo_driver.processgraph.evaluator import evaluate return evaluate(process_graph=process_graph, env=env or self.get_basic_env(), do_dry_run=False) @@ -257,9 +258,10 @@ def get_process_registry(self, api_version: Union[str, ComparableVersion]) -> Pr else: raise OpenEOApiException(message=f"No process support for openEO version {api_version}") - def evaluate(self, process_graph: dict, env: EvalEnv = None): + def evaluate(self, process_graph: dict, env: EvalEnv = None, do_dry_run: Union[bool, DryRunDataTracer] = True): from openeo_driver.processgraph.evaluator import evaluate - return evaluate(process_graph=process_graph, env=env) + + return evaluate(process_graph=process_graph, env=env, do_dry_run=do_dry_run) def validate(self, process_graph: dict, env: EvalEnv = None): from openeo_driver.processgraph.evaluator import evaluate, _collect_end_nodes, convert_node @@ -304,5 +306,3 @@ def validate(self, process_graph: dict, env: EvalEnv = None): def extra_validation(self, process_graph, env, result, source_constraints): return [] - - diff --git a/openeo_driver/views.py b/openeo_driver/views.py index 8a97f0ca..fe34ef07 100644 --- a/openeo_driver/views.py +++ b/openeo_driver/views.py @@ -55,6 +55,7 @@ LINK_REL, ) from openeo_driver.datacube import DriverMlModel +from openeo_driver.dry_run import DryRunDataTracer from openeo_driver.errors import ( FeatureUnsupportedException, FilePathInvalidException, @@ -725,8 +726,10 @@ def result(user: User): } ) + tracer = DryRunDataTracer() + try: - result = backend_implementation.processing.evaluate(process_graph=process_graph, env=env) + result = backend_implementation.processing.evaluate(process_graph=process_graph, env=env, do_dry_run=tracer) _log.info(f"`POST /result`: {type(result)}") if result is None: @@ -743,7 +746,12 @@ def result(user: User): response = result.create_flask_response() costs = backend_implementation.request_costs( - success=True, user=user, request_id=request_id, job_options=job_options + success=True, + user=user, + process_graph=process_graph, + request_id=request_id, + job_options=job_options, + tracer=tracer, ) if costs: # TODO not all costs are accounted for so don't expose in "OpenEO-Costs" yet @@ -752,7 +760,12 @@ def result(user: User): except Exception: # TODO: also send "OpenEO-Costs" header on failure backend_implementation.request_costs( - success=False, user=user, request_id=request_id, job_options=job_options + success=False, + user=user, + process_graph=process_graph, + request_id=request_id, + job_options=job_options, + tracer=tracer, ) raise From 766745c57dfa574f63e5804ecba836cd626a9d80 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Thu, 7 May 2026 11:12:20 +0200 Subject: [PATCH 2/6] adapt API and fix tests https://github.com/Open-EO/openeo-geopyspark-driver/issues/1436 --- openeo_driver/backend.py | 4 ++-- openeo_driver/views.py | 17 ++++++++++------- tests/test_backend.py | 1 + tests/test_views_execute.py | 8 +++++--- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/openeo_driver/backend.py b/openeo_driver/backend.py index 0cc49b2c..980da246 100644 --- a/openeo_driver/backend.py +++ b/openeo_driver/backend.py @@ -1061,11 +1061,11 @@ def request_costs( self, *, user: User, - process_graph: ProcessGraphFlatDict, job_options: Union[dict, None] = None, request_id: str, success: bool, - tracer: DryRunDataTracer, + process_graph: Union[ProcessGraphFlatDict, None] = None, + tracer: Union[DryRunDataTracer, None] = None, ) -> Optional[float]: """ Report resource usage of (current) synchronous processing request and get associated cost. diff --git a/openeo_driver/views.py b/openeo_driver/views.py index 808c6545..57dc8326 100644 --- a/openeo_driver/views.py +++ b/openeo_driver/views.py @@ -729,7 +729,10 @@ def result(user: User): tracer = DryRunDataTracer() try: - result = backend_implementation.processing.evaluate(process_graph=process_graph, env=env, do_dry_run=tracer) + result = backend_implementation.processing.evaluate( + process_graph=copy.deepcopy(process_graph), env=env, do_dry_run=tracer + ) + _log.info(f"`POST /result`: {type(result)}") if result is None: @@ -746,11 +749,11 @@ def result(user: User): response = result.create_flask_response() costs = backend_implementation.request_costs( - success=True, user=user, - process_graph=process_graph, - request_id=request_id, job_options=job_options, + request_id=request_id, + success=True, + process_graph=process_graph, tracer=tracer, ) if costs: @@ -760,11 +763,11 @@ def result(user: User): except Exception: # TODO: also send "OpenEO-Costs" header on failure backend_implementation.request_costs( - success=False, user=user, - process_graph=process_graph, - request_id=request_id, job_options=job_options, + request_id=request_id, + success=False, + process_graph=process_graph, tracer=tracer, ) raise diff --git a/tests/test_backend.py b/tests/test_backend.py index dc86ad22..c6382934 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -20,6 +20,7 @@ LegacyUdfRuntimes, CollectionsListing, ) +from openeo_driver.dry_run import DryRunDataTracer from openeo_driver.errors import CollectionNotFoundException from openeo_driver.users import User diff --git a/tests/test_views_execute.py b/tests/test_views_execute.py index faa522a0..8dd1e1a1 100644 --- a/tests/test_views_execute.py +++ b/tests/test_views_execute.py @@ -22,7 +22,7 @@ from openeo_driver.datacube import DriverDataCube, DriverVectorCube from openeo_driver.datastructs import ResolutionMergeArgs, SarBackscatterArgs -from openeo_driver.dry_run import ProcessType +from openeo_driver.dry_run import ProcessType, DryRunDataTracer from openeo_driver.dummy import dummy_backend from openeo_driver.dummy.dummy_backend import DummyVisitor from openeo_driver.errors import ( @@ -4308,13 +4308,13 @@ def test_vector_buffer_returns_error_on_empty_result_geometry(api): (None, None, None), # request_costs override ( - lambda user, request_id, success, job_options: 1234 + isinstance(user, User), + lambda user, job_options, request_id, success, process_graph, tracer: 1234 + isinstance(user, User), None, "1235", ), # Extra job options handling ( - lambda user, request_id, success, job_options: 1234 * job_options.get("extra", 0), + lambda user, job_options, request_id, success, process_graph, tracer: 1234 * job_options.get("extra", 0), {"extra": 2}, "2468", ), @@ -4363,6 +4363,8 @@ def test_synchronous_processing_request_costs( job_options=job_options, success=success, request_id="r-abc123", + process_graph=pg, + tracer=dirty_equals.IsInstance(DryRunDataTracer), ) From a79a93c08ad36f88c25c39927ef4176b391db50a Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Thu, 7 May 2026 11:48:47 +0200 Subject: [PATCH 3/6] DRY https://github.com/Open-EO/openeo-geopyspark-driver/issues/1436 --- openeo_driver/views.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/openeo_driver/views.py b/openeo_driver/views.py index 57dc8326..48f7e7ae 100644 --- a/openeo_driver/views.py +++ b/openeo_driver/views.py @@ -728,6 +728,15 @@ def result(user: User): tracer = DryRunDataTracer() + request_costs = functools.partial( + backend_implementation.request_costs, + user=user, + job_options=job_options, + request_id=request_id, + process_graph=process_graph, + tracer=tracer, + ) + try: result = backend_implementation.processing.evaluate( process_graph=copy.deepcopy(process_graph), env=env, do_dry_run=tracer @@ -748,28 +757,14 @@ def result(user: User): result = to_save_result(data=result) response = result.create_flask_response() - costs = backend_implementation.request_costs( - user=user, - job_options=job_options, - request_id=request_id, - success=True, - process_graph=process_graph, - tracer=tracer, - ) + costs = request_costs(success=True) if costs: # TODO not all costs are accounted for so don't expose in "OpenEO-Costs" yet response.headers["OpenEO-Costs-experimental"] = costs except Exception: # TODO: also send "OpenEO-Costs" header on failure - backend_implementation.request_costs( - user=user, - job_options=job_options, - request_id=request_id, - success=False, - process_graph=process_graph, - tracer=tracer, - ) + request_costs(success=False) raise # Add request id as "OpenEO-Identifier" like we do for batch jobs. From 120ac199184249ef0def47b151a9327dbd02c52c Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Thu, 7 May 2026 11:52:28 +0200 Subject: [PATCH 4/6] remove unused import https://github.com/Open-EO/openeo-geopyspark-driver/issues/1436 --- tests/test_backend.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index c6382934..dc86ad22 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -20,7 +20,6 @@ LegacyUdfRuntimes, CollectionsListing, ) -from openeo_driver.dry_run import DryRunDataTracer from openeo_driver.errors import CollectionNotFoundException from openeo_driver.users import User From f83a0722606f239ecaecc1ef7ae9280e837f17b1 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Thu, 7 May 2026 11:54:12 +0200 Subject: [PATCH 5/6] adapt version and CHANGELOG https://github.com/Open-EO/openeo-geopyspark-driver/issues/1436 --- CHANGELOG.md | 1 + openeo_driver/_version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 191cdca7..3f1e5a1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and start a new "In Progress" section above it. - Start providing original "node id" to `load_collection` and `load_stac` implementations, e.g. for robust caching puprposes ([#479](https://github.com/Open-EO/openeo-python-driver/issues/479)) - Add (non-standard) `query_stac` process ([#480](https://github.com/Open-EO/openeo-python-driver/pull/480)) - Support experimental `corsa_compress_v2` and `corsa_decompress_v2` processes ([Open-EO/openeo-geotrellis-extensions#702](https://github.com/Open-EO/openeo-geotrellis-extensions/issues/702)) +- Support logging added value for synchronous requests ([Open-EO/openeo-geopyspark-driver#1436](https://github.com/Open-EO/openeo-geopyspark-driver/issues/1436)) ## 0.138.0 diff --git a/openeo_driver/_version.py b/openeo_driver/_version.py index cd2e528f..b8778285 100644 --- a/openeo_driver/_version.py +++ b/openeo_driver/_version.py @@ -1 +1 @@ -__version__ = "0.139.0a7" +__version__ = "0.139.0a8" From 51104a4b12502073656b75902205ec54875d2d66 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Thu, 7 May 2026 12:10:09 +0200 Subject: [PATCH 6/6] up version after merging from master https://github.com/Open-EO/openeo-geopyspark-driver/issues/1436 --- openeo_driver/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeo_driver/_version.py b/openeo_driver/_version.py index b8778285..566f308b 100644 --- a/openeo_driver/_version.py +++ b/openeo_driver/_version.py @@ -1 +1 @@ -__version__ = "0.139.0a8" +__version__ = "0.139.0a9"