From 3e4f44dc912c7244b6ebd38650d678130e2b75e7 Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Wed, 18 Jun 2025 14:33:01 +0200 Subject: [PATCH 01/14] Fix spinner animation with tqdm output and refactor Score-P stream reading - Spinner no longer overlaps with tqdm due to proper threading.Event handling - Score-P stdout and stderr are read in separate threads with chunked reading - Animation is disabled in multicell mode when needed - README.md and error logging improved --- README.md | 2 + src/scorep_jupyter/kernel.py | 168 +++++++++++++++++--------- src/scorep_jupyter/userpersistence.py | 24 ++-- 3 files changed, 123 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 02cae91..5d0fc79 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,8 @@ To see the detailed report for marshalling steps - `SCOREP_JUPYTER_MARSHALLING_D %env SCOREP_JUPYTER_MARSHALLING_DETAILED_REPORT=1 ``` You can disable visual animations shown during long-running tasks by setting the `SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS` environment variable. +This can be useful for debugging, as it ensures that any error messages from your code in cells are shown without being overwritten. +It is also helpful when running code that produces its own progress bars (e.g., using `tqdm`), to prevent output from being obscured. ``` %env SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS=1 ``` diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py index dc93cde..872fb8a 100644 --- a/src/scorep_jupyter/kernel.py +++ b/src/scorep_jupyter/kernel.py @@ -11,6 +11,10 @@ from enum import Enum from textwrap import dedent +from statistics import mean +from typing import IO, AnyStr, Callable + +import pandas as pd from ipykernel.ipkernel import IPythonKernel from scorep_jupyter.userpersistence import PersHelper, scorep_script_name from scorep_jupyter.userpersistence import magics_cleanup, create_busy_spinner @@ -459,6 +463,7 @@ async def scorep_execute( allow_stdin=False, *, cell_id=None, + is_multicell_final=False, ): """ Execute given code with Score-P Python bindings instrumentation. @@ -563,28 +568,7 @@ async def scorep_execute( self.pershelper.postprocess() return reply_status_dump - # Empty cell output, required for interactive output - # e.g. tqdm for-loop progress bar - self.cell_output("\0") - - stdout_lock = threading.Lock() - process_busy_spinner = create_busy_spinner(stdout_lock) - process_busy_spinner.start("Process is running...") - - # Due to splitting into scorep-kernel and ipython extension, - # multicell mode is not supported for coarse-grained measurements - # anymore (in the extension) and we do not show the single cells in - # the ipython extension visualizations after executing them with scorep - # however, since we are using scorep anyway, the ipython extension is - # not useful, since we can count hardware counters anyway - # multicellmode_timestamps = [] - - try: - # multicellmode_timestamps = - self.read_scorep_process_pipe(proc, stdout_lock) - process_busy_spinner.stop("Done.") - except KeyboardInterrupt: - process_busy_spinner.stop("Kernel interrupted.") + multicellmode_timestamps = self.start_reading_scorep_process_streams(proc, is_multicell_final) # In disk mode, subprocess already terminated # after dumping persistence to file @@ -669,67 +653,130 @@ async def scorep_execute( self.pershelper.postprocess() return self.standard_reply() - def read_scorep_process_pipe( - self, proc: subprocess.Popen[bytes], stdout_lock: threading.Lock + def start_reading_scorep_process_streams( + self, + proc: subprocess.Popen[bytes], + is_multicell_final: bool, ) -> list: """ This function reads stdout and stderr of the subprocess running with Score-P instrumentation independently. - It logs all stderr output, collects lines containing - the marker "MCM_TS" (used to identify multi-cell mode timestamps) into - a list, and sends the remaining - stdout lines to the Jupyter cell output. Simultaneous access to stdout is synchronized via a lock to prevent - overlapping with another thread performing + overlapping with stderr reading thread and thread performing long-running process animation. Args: - proc (subprocess.Popen[bytes]): The subprocess whose output is - being read. - stdout_lock (threading.Lock): Lock to avoid output overlapping + proc (subprocess.Popen[bytes]): The subprocess whose output is being read. + is_multicell_final (bool): If multicell mode is finalizing - spinner must be disabled. Returns: list: A list of decoded strings containing "MCM_TS" timestamps. """ + + stdout_lock = threading.Lock() + spinner_stop_event = threading.Event() + process_busy_spinner = create_busy_spinner(stdout_lock, spinner_stop_event, is_multicell_final) + process_busy_spinner.start("Process is running...") + multicellmode_timestamps = [] - sel = selectors.DefaultSelector() - sel.register(proc.stdout, selectors.EVENT_READ) - sel.register(proc.stderr, selectors.EVENT_READ) + # Empty cell output, required for interactive output + # e.g. tqdm for-loop progress bar + self.cell_output("\0") - line_width = 50 - clear_line = "\r" + " " * line_width + "\r" + try: + t_stderr = threading.Thread( + target=self.read_scorep_stderr, + args=( + proc.stderr, + stdout_lock, + spinner_stop_event) + ) + t_stderr.start() - while True: - # Select between stdout and stderr - for key, val in sel.select(): - line = key.fileobj.readline() - if not line: - sel.unregister(key.fileobj) - continue - - decoded_line = line.decode( - sys.getdefaultencoding(), errors="ignore" - ) + multicellmode_timestamps = self.read_scorep_stdout(proc.stdout, stdout_lock, spinner_stop_event) + + t_stderr.join() + process_busy_spinner.stop("Done.") + + except KeyboardInterrupt: + process_busy_spinner.stop("Kernel interrupted.") + + if multicellmode_timestamps: + self.log.debug(f'{multicellmode_timestamps = }') + else: + self.log.debug(f'"multicellmode_timestamps" is empty.') + + + return multicellmode_timestamps + + def read_scorep_stream( + self, + stream: IO[AnyStr], + lock: threading.Lock, + process_line: Callable[[str], None], + read_chunk_size: int = 64, + ): + incomplete_line = "" + endline_pattern = re.compile(r"(.*?[\r\n]|.+$)") - if key.fileobj is proc.stderr: - with stdout_lock: - self.log.warning(f"{decoded_line.strip()}") - elif "MCM_TS" in decoded_line: - multicellmode_timestamps.append(decoded_line) - else: - with stdout_lock: - sys.stdout.write(clear_line) - sys.stdout.flush() - self.cell_output(decoded_line) - # If both stdout and stderr empty -> out of loop - if not sel.get_map(): + + while True: + chunk = stream.read(read_chunk_size) + if not chunk: break + chunk = chunk.decode(sys.getdefaultencoding(), errors="ignore") + lines = endline_pattern.findall(chunk) + if lines: + lines[0] = incomplete_line + lines[0] + if lines[-1][-1] not in ["\n", "\r"]: + incomplete_line = lines.pop(-1) + else: + incomplete_line = "" + for line in lines: + with lock: + process_line(line) + + def read_scorep_stdout( + self, + stdout: IO[AnyStr], + lock: threading.Lock, + spinner_stop_event: threading.Event, + read_chunk_size=64, + ) -> list: + multicellmode_timestamps = [] + line_width = 50 + clear_line = "\r" + " " * line_width + "\r" + def process_stdout_line(line: str): + if "MCM_TS" in line: + multicellmode_timestamps.append(line) + else: + if spinner_stop_event.is_set(): + sys.stdout.write(clear_line) + sys.stdout.flush() + self.cell_output(line) + + self.read_scorep_stream(stdout, lock, process_stdout_line, read_chunk_size) return multicellmode_timestamps + def read_scorep_stderr( + self, + stderr: IO[AnyStr], + lock: threading.Lock, + spinner_stop_event: threading.Event, + read_chunk_size=64, + ): + def process_stderr_line(line: str): + if spinner_stop_event.is_set(): + self.cell_output(line) + self.log.error(line) + + self.read_scorep_stream(stderr, lock, process_stderr_line, read_chunk_size) + + async def do_execute( self, code, @@ -778,6 +825,7 @@ async def do_execute( user_expressions, allow_stdin, cell_id=cell_id, + is_multicell_final=True, ) except Exception: self.cell_output( diff --git a/src/scorep_jupyter/userpersistence.py b/src/scorep_jupyter/userpersistence.py index d56b32c..e18fc0d 100644 --- a/src/scorep_jupyter/userpersistence.py +++ b/src/scorep_jupyter/userpersistence.py @@ -11,6 +11,7 @@ import uuid import importlib + scorep_script_name = "scorep_script.py" @@ -442,7 +443,7 @@ def magics_cleanup(code): class BaseSpinner: - def __init__(self, lock=None): + def __init__(self, lock=None, stop_event=None): pass def _spinner_task(self): @@ -459,22 +460,23 @@ def stop(self, done_message="Done."): class BusySpinner(BaseSpinner): - def __init__(self, lock=None): + def __init__(self, lock=None, stop_event=None): super().__init__(lock) self._lock = lock or threading.Lock() - self._stop_event = threading.Event() + self._stop_event = stop_event or threading.Event() self._thread = threading.Thread(target=self._spinner_task) self.working_message = "" self.done_message = "" def _spinner_task(self): spinner_chars = "|/-\\" + clear_line = " " * 50 idx = 0 while not self._stop_event.is_set(): with self._lock: sys.stdout.write( f"\r{self.working_message} " - f"{spinner_chars[idx % len(spinner_chars)]}" + f"{spinner_chars[idx % len(spinner_chars)]}{clear_line}" ) sys.stdout.flush() time.sleep(0.1) @@ -498,11 +500,11 @@ def stop(self, done_message="Done."): self._thread.join() -def create_busy_spinner(lock=None): - is_enabled = ( - os.getenv("SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS") != "1" - ) - if is_enabled: - return BusySpinner(lock) +def create_busy_spinner(lock=None, stop_event=None, is_multicell_final=False): + is_enabled = os.getenv("SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS") != "1" + if is_enabled and not is_multicell_final: + return BusySpinner(lock, stop_event) else: - return BaseSpinner(lock) + if stop_event: + stop_event.set() + return BaseSpinner() From 00e237ada303ea7ea66899c6f80798a301fa6808 Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Fri, 11 Jul 2025 10:33:02 +0200 Subject: [PATCH 02/14] multicellmode_timestamps removed from reading streams functions --- src/scorep_jupyter/kernel.py | 84 ++++++++++++++---------------------- tests/test_kernel.py | 4 +- 2 files changed, 36 insertions(+), 52 deletions(-) diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py index 872fb8a..ca540e8 100644 --- a/src/scorep_jupyter/kernel.py +++ b/src/scorep_jupyter/kernel.py @@ -568,7 +568,7 @@ async def scorep_execute( self.pershelper.postprocess() return reply_status_dump - multicellmode_timestamps = self.start_reading_scorep_process_streams(proc, is_multicell_final) + self.start_reading_scorep_process_streams(proc, is_multicell_final) # In disk mode, subprocess already terminated # after dumping persistence to file @@ -657,7 +657,7 @@ def start_reading_scorep_process_streams( self, proc: subprocess.Popen[bytes], is_multicell_final: bool, - ) -> list: + ): """ This function reads stdout and stderr of the subprocess running with Score-P instrumentation independently. @@ -679,8 +679,6 @@ def start_reading_scorep_process_streams( process_busy_spinner = create_busy_spinner(stdout_lock, spinner_stop_event, is_multicell_final) process_busy_spinner.start("Process is running...") - multicellmode_timestamps = [] - # Empty cell output, required for interactive output # e.g. tqdm for-loop progress bar self.cell_output("\0") @@ -695,7 +693,7 @@ def start_reading_scorep_process_streams( ) t_stderr.start() - multicellmode_timestamps = self.read_scorep_stdout(proc.stdout, stdout_lock, spinner_stop_event) + self.read_scorep_stdout(proc.stdout, stdout_lock, spinner_stop_event) t_stderr.join() process_busy_spinner.stop("Done.") @@ -703,64 +701,23 @@ def start_reading_scorep_process_streams( except KeyboardInterrupt: process_busy_spinner.stop("Kernel interrupted.") - if multicellmode_timestamps: - self.log.debug(f'{multicellmode_timestamps = }') - else: - self.log.debug(f'"multicellmode_timestamps" is empty.') - - - return multicellmode_timestamps - - def read_scorep_stream( - self, - stream: IO[AnyStr], - lock: threading.Lock, - process_line: Callable[[str], None], - read_chunk_size: int = 64, - ): - incomplete_line = "" - endline_pattern = re.compile(r"(.*?[\r\n]|.+$)") - - - - while True: - chunk = stream.read(read_chunk_size) - if not chunk: - break - chunk = chunk.decode(sys.getdefaultencoding(), errors="ignore") - lines = endline_pattern.findall(chunk) - if lines: - lines[0] = incomplete_line + lines[0] - if lines[-1][-1] not in ["\n", "\r"]: - incomplete_line = lines.pop(-1) - else: - incomplete_line = "" - for line in lines: - with lock: - process_line(line) - def read_scorep_stdout( self, stdout: IO[AnyStr], lock: threading.Lock, spinner_stop_event: threading.Event, read_chunk_size=64, - ) -> list: - multicellmode_timestamps = [] + ): line_width = 50 clear_line = "\r" + " " * line_width + "\r" def process_stdout_line(line: str): - if "MCM_TS" in line: - multicellmode_timestamps.append(line) - else: - if spinner_stop_event.is_set(): - sys.stdout.write(clear_line) - sys.stdout.flush() - self.cell_output(line) + if spinner_stop_event.is_set(): + sys.stdout.write(clear_line) + sys.stdout.flush() + self.cell_output(line) self.read_scorep_stream(stdout, lock, process_stdout_line, read_chunk_size) - return multicellmode_timestamps def read_scorep_stderr( self, @@ -776,6 +733,31 @@ def process_stderr_line(line: str): self.read_scorep_stream(stderr, lock, process_stderr_line, read_chunk_size) + def read_scorep_stream( + self, + stream: IO[AnyStr], + lock: threading.Lock, + process_line: Callable[[str], None], + read_chunk_size: int = 64, + ): + incomplete_line = "" + endline_pattern = re.compile(r"(.*?[\r\n]|.+$)") + + while True: + chunk = stream.read(read_chunk_size) + if not chunk: + break + chunk = chunk.decode(sys.getdefaultencoding(), errors="ignore") + lines = endline_pattern.findall(chunk) + if lines: + lines[0] = incomplete_line + lines[0] + if lines[-1][-1] not in ["\n", "\r"]: + incomplete_line = lines.pop(-1) + else: + incomplete_line = "" + for line in lines: + with lock: + process_line(line) async def do_execute( self, diff --git a/tests/test_kernel.py b/tests/test_kernel.py index 1919bef..8653429 100644 --- a/tests/test_kernel.py +++ b/tests/test_kernel.py @@ -131,7 +131,9 @@ def test_03_persistence(self): self.check_from_notebook("tests/kernel/persistence.ipynb") def test_04_multicell(self): - self.check_from_notebook("tests/kernel/multicell.ipynb") + pass + # TODO: should be moved to the extension or tested only if extension is loaded + # self.check_from_notebook("tests/kernel/multicell.ipynb") def test_05_writemode(self): self.check_from_notebook("tests/kernel/writemode.ipynb") From 23ad7c87aa1e7e182de549c800d0bc4d51afd9a0 Mon Sep 17 00:00:00 2001 From: Elias Werner Date: Mon, 14 Jul 2025 10:20:51 +0200 Subject: [PATCH 03/14] remove MCM from child process, check for multiple truthy values --- src/scorep_jupyter/kernel.py | 16 +++++++++------- src/scorep_jupyter/userpersistence.py | 3 ++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py index ca540e8..2debd0e 100644 --- a/src/scorep_jupyter/kernel.py +++ b/src/scorep_jupyter/kernel.py @@ -238,7 +238,6 @@ def append_multicellmode(self, code): f"print('Executing cell {self.multicell_cellcount}')\n" + f"print('''{code}''')\n" + f"print('-' * {max_line_len})\n" - + "print('MCM_TS'+str(time.time()))\n" + f"{code}\n" + "print('''\n''')\n" ) @@ -667,16 +666,18 @@ def start_reading_scorep_process_streams( long-running process animation. Args: - proc (subprocess.Popen[bytes]): The subprocess whose output is being read. - is_multicell_final (bool): If multicell mode is finalizing - spinner must be disabled. + proc (subprocess.Popen[bytes]): The subprocess whose output is + being read. + is_multicell_final (bool): If multicell mode is finalizing - + spinner must be disabled. - Returns: - list: A list of decoded strings containing "MCM_TS" timestamps. """ stdout_lock = threading.Lock() spinner_stop_event = threading.Event() - process_busy_spinner = create_busy_spinner(stdout_lock, spinner_stop_event, is_multicell_final) + process_busy_spinner = create_busy_spinner(stdout_lock, + spinner_stop_event, + is_multicell_final) process_busy_spinner.start("Process is running...") # Empty cell output, required for interactive output @@ -693,7 +694,8 @@ def start_reading_scorep_process_streams( ) t_stderr.start() - self.read_scorep_stdout(proc.stdout, stdout_lock, spinner_stop_event) + self.read_scorep_stdout(proc.stdout, stdout_lock, + spinner_stop_event) t_stderr.join() process_busy_spinner.stop("Done.") diff --git a/src/scorep_jupyter/userpersistence.py b/src/scorep_jupyter/userpersistence.py index e18fc0d..9dae9bf 100644 --- a/src/scorep_jupyter/userpersistence.py +++ b/src/scorep_jupyter/userpersistence.py @@ -501,7 +501,8 @@ def stop(self, done_message="Done."): def create_busy_spinner(lock=None, stop_event=None, is_multicell_final=False): - is_enabled = os.getenv("SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS") != "1" + is_enabled = (os.getenv("SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS") + .lower() not in ['true', '1', 't']) if is_enabled and not is_multicell_final: return BusySpinner(lock, stop_event) else: From 372324f7da42008f5dfd25d7e4e0c074debd6cef Mon Sep 17 00:00:00 2001 From: Elias Werner Date: Mon, 14 Jul 2025 10:52:17 +0200 Subject: [PATCH 04/14] formatting, linter and bugfix in truthy values eval --- src/scorep_jupyter/kernel.py | 72 +++++++++++++-------------- src/scorep_jupyter/userpersistence.py | 6 ++- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py index 2debd0e..47413df 100644 --- a/src/scorep_jupyter/kernel.py +++ b/src/scorep_jupyter/kernel.py @@ -1,33 +1,29 @@ import datetime +import importlib +import logging.config import os import re -import selectors +import shutil import subprocess import sys import threading import time -import shutil -import logging.config - from enum import Enum from textwrap import dedent -from statistics import mean from typing import IO, AnyStr, Callable -import pandas as pd from ipykernel.ipkernel import IPythonKernel -from scorep_jupyter.userpersistence import PersHelper, scorep_script_name -from scorep_jupyter.userpersistence import magics_cleanup, create_busy_spinner -import importlib + from scorep_jupyter.kernel_messages import ( KernelErrorCode, KERNEL_ERROR_MESSAGES, ) +from scorep_jupyter.userpersistence import PersHelper, scorep_script_name +from scorep_jupyter.userpersistence import magics_cleanup, create_busy_spinner +from .logging_config import LOGGING # import scorep_jupyter.multinode_monitor.slurm_monitor as slurm_monitor -from .logging_config import LOGGING - PYTHON_EXECUTABLE = sys.executable userpersistence_token = "scorep_jupyter.userpersistence" jupyter_dump = "jupyter_dump.pkl" @@ -675,9 +671,9 @@ def start_reading_scorep_process_streams( stdout_lock = threading.Lock() spinner_stop_event = threading.Event() - process_busy_spinner = create_busy_spinner(stdout_lock, - spinner_stop_event, - is_multicell_final) + process_busy_spinner = create_busy_spinner( + stdout_lock, spinner_stop_event, is_multicell_final + ) process_busy_spinner.start("Process is running...") # Empty cell output, required for interactive output @@ -687,15 +683,13 @@ def start_reading_scorep_process_streams( try: t_stderr = threading.Thread( target=self.read_scorep_stderr, - args=( - proc.stderr, - stdout_lock, - spinner_stop_event) + args=(proc.stderr, stdout_lock, spinner_stop_event), ) t_stderr.start() - self.read_scorep_stdout(proc.stdout, stdout_lock, - spinner_stop_event) + self.read_scorep_stdout( + proc.stdout, stdout_lock, spinner_stop_event + ) t_stderr.join() process_busy_spinner.stop("Done.") @@ -704,11 +698,11 @@ def start_reading_scorep_process_streams( process_busy_spinner.stop("Kernel interrupted.") def read_scorep_stdout( - self, - stdout: IO[AnyStr], - lock: threading.Lock, - spinner_stop_event: threading.Event, - read_chunk_size=64, + self, + stdout: IO[AnyStr], + lock: threading.Lock, + spinner_stop_event: threading.Event, + read_chunk_size=64, ): line_width = 50 clear_line = "\r" + " " * line_width + "\r" @@ -719,28 +713,32 @@ def process_stdout_line(line: str): sys.stdout.flush() self.cell_output(line) - self.read_scorep_stream(stdout, lock, process_stdout_line, read_chunk_size) + self.read_scorep_stream( + stdout, lock, process_stdout_line, read_chunk_size + ) def read_scorep_stderr( - self, - stderr: IO[AnyStr], - lock: threading.Lock, - spinner_stop_event: threading.Event, - read_chunk_size=64, + self, + stderr: IO[AnyStr], + lock: threading.Lock, + spinner_stop_event: threading.Event, + read_chunk_size=64, ): def process_stderr_line(line: str): if spinner_stop_event.is_set(): self.cell_output(line) self.log.error(line) - self.read_scorep_stream(stderr, lock, process_stderr_line, read_chunk_size) + self.read_scorep_stream( + stderr, lock, process_stderr_line, read_chunk_size + ) def read_scorep_stream( - self, - stream: IO[AnyStr], - lock: threading.Lock, - process_line: Callable[[str], None], - read_chunk_size: int = 64, + self, + stream: IO[AnyStr], + lock: threading.Lock, + process_line: Callable[[str], None], + read_chunk_size: int = 64, ): incomplete_line = "" endline_pattern = re.compile(r"(.*?[\r\n]|.+$)") diff --git a/src/scorep_jupyter/userpersistence.py b/src/scorep_jupyter/userpersistence.py index 9dae9bf..666ea7a 100644 --- a/src/scorep_jupyter/userpersistence.py +++ b/src/scorep_jupyter/userpersistence.py @@ -501,8 +501,10 @@ def stop(self, done_message="Done."): def create_busy_spinner(lock=None, stop_event=None, is_multicell_final=False): - is_enabled = (os.getenv("SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS") - .lower() not in ['true', '1', 't']) + + is_enabled = str(os.getenv( + "SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS" + )).lower() not in ["true", "1", "t"] if is_enabled and not is_multicell_final: return BusySpinner(lock, stop_event) else: From 04bc5a900a7389dbb8f7afe2a9d692dbb572386f Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Mon, 14 Jul 2025 18:48:30 +0200 Subject: [PATCH 05/14] logging in files fixed, minor stream reading improvements --- src/scorep_jupyter/kernel.py | 17 +++++++++-------- src/scorep_jupyter/logging_config.py | 11 ++++++----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py index 47413df..0295847 100644 --- a/src/scorep_jupyter/kernel.py +++ b/src/scorep_jupyter/kernel.py @@ -674,28 +674,29 @@ def start_reading_scorep_process_streams( process_busy_spinner = create_busy_spinner( stdout_lock, spinner_stop_event, is_multicell_final ) - process_busy_spinner.start("Process is running...") + t_stderr = threading.Thread( + target=self.read_scorep_stderr, + args=(proc.stderr, stdout_lock, spinner_stop_event), + ) # Empty cell output, required for interactive output # e.g. tqdm for-loop progress bar self.cell_output("\0") try: - t_stderr = threading.Thread( - target=self.read_scorep_stderr, - args=(proc.stderr, stdout_lock, spinner_stop_event), - ) + process_busy_spinner.start("Process is running...") t_stderr.start() self.read_scorep_stdout( proc.stdout, stdout_lock, spinner_stop_event ) - t_stderr.join() process_busy_spinner.stop("Done.") except KeyboardInterrupt: process_busy_spinner.stop("Kernel interrupted.") + finally: + t_stderr.join() def read_scorep_stdout( self, @@ -727,7 +728,7 @@ def read_scorep_stderr( def process_stderr_line(line: str): if spinner_stop_event.is_set(): self.cell_output(line) - self.log.error(line) + self.log.error(line.strip()) self.read_scorep_stream( stderr, lock, process_stderr_line, read_chunk_size @@ -909,7 +910,7 @@ def log_error(self, code: KernelErrorCode, **kwargs): ) message = template.format(mode=mode, marshaller=marshaller, **kwargs) - self.log.error(message) + self.log.error(message.strip()) self.cell_output("KernelError: " + message, "stderr") diff --git a/src/scorep_jupyter/logging_config.py b/src/scorep_jupyter/logging_config.py index f6f8c54..3f9e900 100644 --- a/src/scorep_jupyter/logging_config.py +++ b/src/scorep_jupyter/logging_config.py @@ -1,9 +1,10 @@ import logging import os import sys +from pathlib import Path -LOGGING_DIR = "logging" +LOGGING_DIR = Path().cwd().parent / "logging" os.makedirs(LOGGING_DIR, exist_ok=True) @@ -17,7 +18,7 @@ def filter(self, record): return record.levelno < logging.ERROR -class scorep_jupyterKernelOnlyFilter(logging.Filter): +class ScorepJupyterKernelOnlyFilter(logging.Filter): def filter(self, record): return "scorep_jupyter" in record.pathname @@ -55,8 +56,8 @@ def filter(self, record): "class": "logging.StreamHandler", "stream": sys.stdout, "filters": [ - "ignore_error_filter", # prevents from writing to jupyter - # cell output twice + # prevents from writing to jupyter cell output twice + "ignore_error_filter", "scorep_jupyter_kernel_only_filter", ], }, @@ -65,7 +66,7 @@ def filter(self, record): "jupyter_filter": {"()": JupyterLogFilter}, "ignore_error_filter": {"()": IgnoreErrorFilter}, "scorep_jupyter_kernel_only_filter": { - "()": scorep_jupyterKernelOnlyFilter + "()": ScorepJupyterKernelOnlyFilter }, }, "root": { From 57e52a4ccad223c66dd906a8697ea23d8a37941b Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Mon, 14 Jul 2025 19:49:36 +0200 Subject: [PATCH 06/14] disabling animations hint added --- src/scorep_jupyter/kernel.py | 18 ++++++++++++++++++ src/scorep_jupyter/kernel_messages.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py index 0295847..adc4d2e 100644 --- a/src/scorep_jupyter/kernel.py +++ b/src/scorep_jupyter/kernel.py @@ -586,10 +586,26 @@ async def scorep_execute( allow_stdin=allow_stdin, cell_id=cell_id, ) + + is_spinner_enabled = str(os.getenv( + "SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS" + )).lower() not in ["true", "1", "t"] + if is_spinner_enabled: + scorep_process_error_hint = ( + "\nHint: If the animation spinner is active, " + "runtime errors in Score-P cells might be hidden.\n" + "Try disabling the spinner with " + "%env SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS=1 " + "and/or check the log file for details." + ) + else: + scorep_process_error_hint = "" + if reply_status_update["status"] != "ok": self.log_error( KernelErrorCode.PERSISTENCE_LOAD_FAIL, direction="Score-P -> Jupyter", + optional_hint = scorep_process_error_hint ) self.pershelper.postprocess() return reply_status_update @@ -602,6 +618,8 @@ async def scorep_execute( self.log_error( KernelErrorCode.PERSISTENCE_LOAD_FAIL, direction="Score-P -> Jupyter", + optional_hint = scorep_process_error_hint + ) return self.standard_reply() diff --git a/src/scorep_jupyter/kernel_messages.py b/src/scorep_jupyter/kernel_messages.py index 98ba5fd..ed12ab8 100644 --- a/src/scorep_jupyter/kernel_messages.py +++ b/src/scorep_jupyter/kernel_messages.py @@ -29,7 +29,7 @@ class KernelErrorCode(Enum): ), KernelErrorCode.PERSISTENCE_LOAD_FAIL: ( "[mode: {mode}] Failed to load persistence " - "({direction}, marshaller: {marshaller})." + "({direction}, marshaller: {marshaller}). {optional_hint}" ), KernelErrorCode.SCOREP_SUBPROCESS_FAIL: ( "[mode: {mode}] Subprocess terminated unexpectedly. " From 281508572c535dc79c250edcc4e29615a5134bd0 Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Mon, 14 Jul 2025 21:30:59 +0200 Subject: [PATCH 07/14] getting stuck while reading pipes fixed --- src/scorep_jupyter/kernel.py | 66 +++++++++++++++--------------------- 1 file changed, 27 insertions(+), 39 deletions(-) diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py index adc4d2e..da05da8 100644 --- a/src/scorep_jupyter/kernel.py +++ b/src/scorep_jupyter/kernel.py @@ -565,17 +565,16 @@ async def scorep_execute( self.start_reading_scorep_process_streams(proc, is_multicell_final) - # In disk mode, subprocess already terminated - # after dumping persistence to file - if self.pershelper.mode == "disk": - if proc.returncode: - self.pershelper.postprocess() - self.cell_output( - "KernelError: Cell execution failed, cell persistence " - "was not recorded.", - "stderr", - ) - return self.standard_reply() + if proc.poll(): + self.pershelper.postprocess() + self.log_error( + KernelErrorCode.PERSISTENCE_LOAD_FAIL, + direction="Score-P -> Jupyter", + optional_hint = self.get_scorep_process_error_hint() + + ) + return self.standard_reply() + # Ghost cell - load subprocess persistence back to Jupyter notebook # Run in a "silent" way to not increase cells counter reply_status_update = await super().do_execute( @@ -587,42 +586,15 @@ async def scorep_execute( cell_id=cell_id, ) - is_spinner_enabled = str(os.getenv( - "SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS" - )).lower() not in ["true", "1", "t"] - if is_spinner_enabled: - scorep_process_error_hint = ( - "\nHint: If the animation spinner is active, " - "runtime errors in Score-P cells might be hidden.\n" - "Try disabling the spinner with " - "%env SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS=1 " - "and/or check the log file for details." - ) - else: - scorep_process_error_hint = "" - if reply_status_update["status"] != "ok": self.log_error( KernelErrorCode.PERSISTENCE_LOAD_FAIL, direction="Score-P -> Jupyter", - optional_hint = scorep_process_error_hint + optional_hint = self.get_scorep_process_error_hint() ) self.pershelper.postprocess() return reply_status_update - # In memory mode, subprocess terminates once jupyter_update is - # executed and pipe is closed - if self.pershelper.mode == "memory": - if proc.poll(): - self.pershelper.postprocess() - self.log_error( - KernelErrorCode.PERSISTENCE_LOAD_FAIL, - direction="Score-P -> Jupyter", - optional_hint = scorep_process_error_hint - - ) - return self.standard_reply() - # Determine directory to which trace files were saved by Score-P scorep_folder = "" if "SCOREP_EXPERIMENT_DIRECTORY" in os.environ: @@ -778,6 +750,22 @@ def read_scorep_stream( with lock: process_line(line) + @staticmethod + def get_scorep_process_error_hint(): + is_spinner_enabled = str(os.getenv( + "SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS" + )).lower() not in ["true", "1", "t"] + scorep_process_error_hint = "" + if is_spinner_enabled: + scorep_process_error_hint = ( + "\nHint: If the animation spinner is active, " + "runtime errors in Score-P cells might be hidden.\n" + "Try disabling the spinner with " + "%env SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS=1 " + "and/or check the log file for details." + ) + return scorep_process_error_hint + async def do_execute( self, code, From 748f4efe94aa635dbfb959293ce551d685c770d5 Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Mon, 14 Jul 2025 21:37:54 +0200 Subject: [PATCH 08/14] KernelTestLogError fix --- tests/test_kernel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_kernel.py b/tests/test_kernel.py index 8653429..0aaab6a 100644 --- a/tests/test_kernel.py +++ b/tests/test_kernel.py @@ -185,6 +185,7 @@ def test_error_templates_are_formatable(self): "direction": "dummy_direction", "detail": "dummy_detail", "step": "dummy_step", + "optional_hint": "dummy_optional_hint", } for code, template in KERNEL_ERROR_MESSAGES.items(): From e314bce3860ceed509bb535f1dee4a4b62b4ef31 Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Tue, 15 Jul 2025 13:54:13 +0200 Subject: [PATCH 09/14] hint message improved; logging dir renamed to logs_scorep_jupyter --- .gitignore | 1 + src/scorep_jupyter/kernel.py | 21 +++------------------ src/scorep_jupyter/kernel_messages.py | 20 ++++++++++++++++++++ src/scorep_jupyter/logging_config.py | 2 +- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index dda875d..c666fda 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ __pycache__/ .userpersistency build logging +kernel_logs **/*.egg-info diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py index da05da8..36c1116 100644 --- a/src/scorep_jupyter/kernel.py +++ b/src/scorep_jupyter/kernel.py @@ -17,6 +17,7 @@ from scorep_jupyter.kernel_messages import ( KernelErrorCode, KERNEL_ERROR_MESSAGES, + get_scorep_process_error_hint, ) from scorep_jupyter.userpersistence import PersHelper, scorep_script_name from scorep_jupyter.userpersistence import magics_cleanup, create_busy_spinner @@ -570,7 +571,7 @@ async def scorep_execute( self.log_error( KernelErrorCode.PERSISTENCE_LOAD_FAIL, direction="Score-P -> Jupyter", - optional_hint = self.get_scorep_process_error_hint() + optional_hint = get_scorep_process_error_hint() ) return self.standard_reply() @@ -590,7 +591,7 @@ async def scorep_execute( self.log_error( KernelErrorCode.PERSISTENCE_LOAD_FAIL, direction="Score-P -> Jupyter", - optional_hint = self.get_scorep_process_error_hint() + optional_hint = get_scorep_process_error_hint() ) self.pershelper.postprocess() return reply_status_update @@ -750,22 +751,6 @@ def read_scorep_stream( with lock: process_line(line) - @staticmethod - def get_scorep_process_error_hint(): - is_spinner_enabled = str(os.getenv( - "SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS" - )).lower() not in ["true", "1", "t"] - scorep_process_error_hint = "" - if is_spinner_enabled: - scorep_process_error_hint = ( - "\nHint: If the animation spinner is active, " - "runtime errors in Score-P cells might be hidden.\n" - "Try disabling the spinner with " - "%env SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS=1 " - "and/or check the log file for details." - ) - return scorep_process_error_hint - async def do_execute( self, code, diff --git a/src/scorep_jupyter/kernel_messages.py b/src/scorep_jupyter/kernel_messages.py index ed12ab8..a03283d 100644 --- a/src/scorep_jupyter/kernel_messages.py +++ b/src/scorep_jupyter/kernel_messages.py @@ -1,5 +1,8 @@ +import os from enum import Enum, auto +from .logging_config import LOGGING + class KernelErrorCode(Enum): PERSISTENCE_SETUP_FAIL = auto() @@ -36,3 +39,20 @@ class KernelErrorCode(Enum): "Persistence not recorded (marshaller: {marshaller})." ), } + + +def get_scorep_process_error_hint(): + is_spinner_enabled = str(os.getenv( + "SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS" + )).lower() not in ["true", "1", "t"] + scorep_process_error_hint = "" + if is_spinner_enabled: + scorep_process_error_hint = ( + "\nHint: If the animation spinner is active, " + "runtime errors in Score-P cells might be hidden.\n" + "Try disabling the spinner with " + "%env SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS=1 " + f"and/or check the log: " + f"{LOGGING['handlers']['error_file']['filename']} for details." + ) + return scorep_process_error_hint diff --git a/src/scorep_jupyter/logging_config.py b/src/scorep_jupyter/logging_config.py index 3f9e900..5af9446 100644 --- a/src/scorep_jupyter/logging_config.py +++ b/src/scorep_jupyter/logging_config.py @@ -4,7 +4,7 @@ from pathlib import Path -LOGGING_DIR = Path().cwd().parent / "logging" +LOGGING_DIR = Path().cwd().parent / "logs_scorep_jupyter" os.makedirs(LOGGING_DIR, exist_ok=True) From 0d7ed5692b2612494799be5e1bfb8fd503d834c3 Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Mon, 21 Jul 2025 17:02:24 +0200 Subject: [PATCH 10/14] make log directory relative to project root --- src/scorep_jupyter/logging_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scorep_jupyter/logging_config.py b/src/scorep_jupyter/logging_config.py index 5af9446..be14c71 100644 --- a/src/scorep_jupyter/logging_config.py +++ b/src/scorep_jupyter/logging_config.py @@ -4,7 +4,8 @@ from pathlib import Path -LOGGING_DIR = Path().cwd().parent / "logs_scorep_jupyter" +PROJECT_ROOT = Path(__file__).resolve().parent.parent +LOGGING_DIR = PROJECT_ROOT / "logs_scorep_jupyter" os.makedirs(LOGGING_DIR, exist_ok=True) From cf6091b240b501bd20b6d4e5176c5aa1b78c411b Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Tue, 22 Jul 2025 13:24:55 +0200 Subject: [PATCH 11/14] error logs without suppressing and Logging Configuration section in README.md update --- README.md | 14 +++++++++++++- src/scorep_jupyter/kernel.py | 3 ++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5d0fc79..f5b808f 100644 --- a/README.md +++ b/README.md @@ -207,9 +207,21 @@ When dealing with big data structures, there might be a big runtime overhead at ## Logging Configuration To adjust logging and obtain more detailed output about the behavior of the scorep_jupyter kernel, refer to the `src/logging_config.py` file. - This file contains configuration options for controlling the verbosity, format, and destination of log messages. You can customize it to suit your debugging needs. +Log files are stored in the following directory: +``` +scorep_jupyter_kernel_python/ +├── logs_scorep_jupyter/ +│ ├── debug.log +│ ├── info.log +└── └── error.log +``` +In some cases, you may want to suppress tqdm messages that are saved to error.log (since tqdm outputs to stderr). This can be done using the following environment variable: +``` +%env TQDM_DISABLE=1 +``` + # Future Work The kernel is still under development. diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py index 36c1116..b5aa4b8 100644 --- a/src/scorep_jupyter/kernel.py +++ b/src/scorep_jupyter/kernel.py @@ -717,9 +717,10 @@ def read_scorep_stderr( read_chunk_size=64, ): def process_stderr_line(line: str): + self.log.error(line.strip()) if spinner_stop_event.is_set(): self.cell_output(line) - self.log.error(line.strip()) + self.read_scorep_stream( stderr, lock, process_stderr_line, read_chunk_size From 0d4d863355cea421abddc9678ba6b08b226e6cb3 Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Fri, 25 Jul 2025 13:16:15 +0200 Subject: [PATCH 12/14] logs directory name changed --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index c666fda..dd4f63d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,5 @@ __pycache__/ .variables .userpersistency build -logging -kernel_logs +logs_scorep_jupyter/ **/*.egg-info From 95a8e973b1da3efdc96283a5e2a59eaf3706f8cc Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Mon, 28 Jul 2025 13:35:04 +0200 Subject: [PATCH 13/14] preserve subprocess output in variables to show later; hint message corrected --- src/scorep_jupyter/kernel.py | 48 +++++++++++++++++++++------ src/scorep_jupyter/kernel_messages.py | 8 ++--- src/scorep_jupyter/logging_config.py | 4 ++- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py index b5aa4b8..3d9a537 100644 --- a/src/scorep_jupyter/kernel.py +++ b/src/scorep_jupyter/kernel.py @@ -10,7 +10,7 @@ import time from enum import Enum from textwrap import dedent -from typing import IO, AnyStr, Callable +from typing import IO, AnyStr, Callable, List, TextIO from ipykernel.ipkernel import IPythonKernel @@ -572,7 +572,6 @@ async def scorep_execute( KernelErrorCode.PERSISTENCE_LOAD_FAIL, direction="Score-P -> Jupyter", optional_hint = get_scorep_process_error_hint() - ) return self.standard_reply() @@ -665,29 +664,37 @@ def start_reading_scorep_process_streams( process_busy_spinner = create_busy_spinner( stdout_lock, spinner_stop_event, is_multicell_final ) + + captured_stdout: List[str] = [] + captured_stderr: List[str] = [] # Output parameter (return not possible from thread) t_stderr = threading.Thread( target=self.read_scorep_stderr, - args=(proc.stderr, stdout_lock, spinner_stop_event), + args=(proc.stderr, stdout_lock, spinner_stop_event, captured_stderr), ) # Empty cell output, required for interactive output # e.g. tqdm for-loop progress bar self.cell_output("\0") + spinner_message = "Done." + try: process_busy_spinner.start("Process is running...") t_stderr.start() - self.read_scorep_stdout( + captured_stdout = self.read_scorep_stdout( proc.stdout, stdout_lock, spinner_stop_event ) - process_busy_spinner.stop("Done.") - except KeyboardInterrupt: - process_busy_spinner.stop("Kernel interrupted.") + spinner_message = "Kernel interrupted." finally: t_stderr.join() + process_busy_spinner.stop(spinner_message) + + # Handle recorded output (in case if it is suppressed by spinner animation) + self.handle_captured_output(captured_stdout, stream="stdout") + self.handle_captured_output(captured_stderr, stream="stderr") def read_scorep_stdout( self, @@ -695,32 +702,40 @@ def read_scorep_stdout( lock: threading.Lock, spinner_stop_event: threading.Event, read_chunk_size=64, - ): + ) -> List[str]: line_width = 50 clear_line = "\r" + " " * line_width + "\r" + captured_stdout: List[str] = [] + def process_stdout_line(line: str): if spinner_stop_event.is_set(): sys.stdout.write(clear_line) sys.stdout.flush() self.cell_output(line) + else: + captured_stdout.append(line) self.read_scorep_stream( stdout, lock, process_stdout_line, read_chunk_size ) + return captured_stdout def read_scorep_stderr( self, stderr: IO[AnyStr], lock: threading.Lock, spinner_stop_event: threading.Event, + captured_stderr: List[str], read_chunk_size=64, ): + def process_stderr_line(line: str): - self.log.error(line.strip()) if spinner_stop_event.is_set(): - self.cell_output(line) - + self.log.error(line.strip()) + self.cell_output(line, 'stderr') + else: + captured_stderr.append(line) self.read_scorep_stream( stderr, lock, process_stderr_line, read_chunk_size @@ -752,6 +767,17 @@ def read_scorep_stream( with lock: process_line(line) + def handle_captured_output(self, output: List[str], stream: str): + if output: + text_output = "".join(output) + if stream == "stdout": + self.cell_output(text_output, stream=stream) + elif stream == "stderr": + self.cell_output(text_output, stream=stream) + self.log.error(text_output) + else: + self.log.error(f"Undefined stream type: {stream}") + async def do_execute( self, code, diff --git a/src/scorep_jupyter/kernel_messages.py b/src/scorep_jupyter/kernel_messages.py index a03283d..d0bd186 100644 --- a/src/scorep_jupyter/kernel_messages.py +++ b/src/scorep_jupyter/kernel_messages.py @@ -48,11 +48,7 @@ def get_scorep_process_error_hint(): scorep_process_error_hint = "" if is_spinner_enabled: scorep_process_error_hint = ( - "\nHint: If the animation spinner is active, " - "runtime errors in Score-P cells might be hidden.\n" - "Try disabling the spinner with " - "%env SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS=1 " - f"and/or check the log: " - f"{LOGGING['handlers']['error_file']['filename']} for details." + "\nHint: full error info saved to log file: " + f"{LOGGING['handlers']['error_file']['filename']}" ) return scorep_process_error_hint diff --git a/src/scorep_jupyter/logging_config.py b/src/scorep_jupyter/logging_config.py index be14c71..5e73cd2 100644 --- a/src/scorep_jupyter/logging_config.py +++ b/src/scorep_jupyter/logging_config.py @@ -4,7 +4,9 @@ from pathlib import Path -PROJECT_ROOT = Path(__file__).resolve().parent.parent +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +print(f'{Path(__file__).as_uri()=}') +print(f'{PROJECT_ROOT=}') LOGGING_DIR = PROJECT_ROOT / "logs_scorep_jupyter" os.makedirs(LOGGING_DIR, exist_ok=True) From d6fa2da97c458767b840d7101d85d0454a30a15f Mon Sep 17 00:00:00 2001 From: OutlyingWest Date: Mon, 28 Jul 2025 13:40:42 +0200 Subject: [PATCH 14/14] hint logic changed to be shown in all cases --- src/scorep_jupyter/kernel_messages.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/scorep_jupyter/kernel_messages.py b/src/scorep_jupyter/kernel_messages.py index d0bd186..bd5a3fa 100644 --- a/src/scorep_jupyter/kernel_messages.py +++ b/src/scorep_jupyter/kernel_messages.py @@ -42,13 +42,8 @@ class KernelErrorCode(Enum): def get_scorep_process_error_hint(): - is_spinner_enabled = str(os.getenv( - "SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS" - )).lower() not in ["true", "1", "t"] - scorep_process_error_hint = "" - if is_spinner_enabled: - scorep_process_error_hint = ( - "\nHint: full error info saved to log file: " - f"{LOGGING['handlers']['error_file']['filename']}" - ) + scorep_process_error_hint = ( + "\nHint: full error info saved to log file: " + f"{LOGGING['handlers']['error_file']['filename']}" + ) return scorep_process_error_hint