From dfe64b4909dee62d3cabb1eb578300411c025bca Mon Sep 17 00:00:00 2001 From: Daniel Kohler <11864045+ddkohler@users.noreply.github.com> Date: Mon, 20 Oct 2025 15:23:24 -0500 Subject: [PATCH 1/9] refactor asyncio loop --- yaqd-core/yaqd_core/_is_daemon.py | 45 +++++++++++++++++-------------- yaqd-core/yaqd_core/_protocol.py | 2 +- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/yaqd-core/yaqd_core/_is_daemon.py b/yaqd-core/yaqd_core/_is_daemon.py index 57aa83c..a93615d 100755 --- a/yaqd-core/yaqd_core/_is_daemon.py +++ b/yaqd-core/yaqd_core/_is_daemon.py @@ -123,17 +123,6 @@ def _traits(cls) -> List[str]: @classmethod def main(cls): - """Run the event loop.""" - loop = asyncio.get_event_loop() - if sys.platform.startswith("win"): - signals = () - else: - signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) - for s in signals: - loop.add_signal_handler( - s, lambda s=s: asyncio.create_task(cls.shutdown_all(s, loop)) - ) - parser = argparse.ArgumentParser() parser.add_argument( "--config", @@ -199,19 +188,28 @@ def main(cls): with open(config_filepath, "rb") as f: config_file = tomli.load(f) - loop.create_task(cls._main(config_filepath, config_file, args)) + # Run the event loop try: - loop.run_forever() + asyncio.run( + cls._main(config_filepath, config_file, args) + ) except asyncio.CancelledError: pass - finally: - loop.close() @classmethod async def _main(cls, config_filepath, config_file, args=None): """Parse command line arguments, start event loop tasks.""" loop = asyncio.get_running_loop() - cls.__servers = [] + if sys.platform.startswith("win"): + signals = () + else: + signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) + for s in signals: + loop.add_signal_handler( + s, lambda s=s: asyncio.create_task(cls.shutdown_all(s, loop)) + ) + + cls.__servers = set() for section in config_file: if section == "shared-settings": continue @@ -225,7 +223,7 @@ async def _main(cls, config_filepath, config_file, args=None): while cls.__servers: awaiting = cls.__servers - cls.__servers = [] + cls.__servers = set() await asyncio.wait(awaiting) await asyncio.sleep(1) loop.stop() @@ -252,7 +250,11 @@ def server(daemon): server(daemon), config.get("host", ""), config.get("port", None) ) daemon._server = ser - cls.__servers.append(asyncio.create_task(ser.serve_forever())) + # cls.__servers.add(asyncio.create_task()) + task = asyncio.create_task(ser.serve_forever()) + cls.__servers.add(task) + # Add a done callback to remove the task from the set when it completes + task.add_done_callback(cls.__servers.discard) @classmethod def _parse_config(cls, config_file, section, args=None): @@ -300,13 +302,16 @@ async def shutdown_all(cls, sig, loop): tasks = [ t for t in asyncio.all_tasks() if ( - t is not asyncio.current_task() - and "serve_forever" not in t.get_coro().__repr__() + t is not asyncio.current_task() + # and "serve_forever" not in t.get_coro().__repr__() ) ] + logger.info("gathering") for task in tasks: logger.info(task.get_coro()) await asyncio.gather(*tasks, return_exceptions=True) + logger.info("gathered") + logger.info([task.get_coro() for task in asyncio.all_tasks()]) [d._save_state() for d in cls._daemons] if hasattr(signal, "SIGHUP") and sig == signal.SIGHUP: config_filepath = [d._config_filepath for d in cls._daemons][0] diff --git a/yaqd-core/yaqd_core/_protocol.py b/yaqd-core/yaqd_core/_protocol.py index 6644a08..10bd3fa 100644 --- a/yaqd-core/yaqd_core/_protocol.py +++ b/yaqd-core/yaqd_core/_protocol.py @@ -28,7 +28,7 @@ def connection_made(self, transport): self.transport = transport self.unpacker = avrorpc.Unpacker(self._avro_protocol) self._daemon._connection_made(peername) - self.task = asyncio.get_event_loop().create_task(self.process_requests()) + self.task = asyncio.get_running_loop().create_task(self.process_requests()) def data_received(self, data): """Process an incomming request.""" From f97826cb9193bfab148e32f3e82d1b467c5d5ab0 Mon Sep 17 00:00:00 2001 From: Daniel Kohler <11864045+ddkohler@users.noreply.github.com> Date: Mon, 20 Oct 2025 15:38:20 -0500 Subject: [PATCH 2/9] Update _fake_sensor.py --- yaqd-fakes/yaqd_fakes/_fake_sensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yaqd-fakes/yaqd_fakes/_fake_sensor.py b/yaqd-fakes/yaqd_fakes/_fake_sensor.py index c56f1b3..980115e 100644 --- a/yaqd-fakes/yaqd_fakes/_fake_sensor.py +++ b/yaqd-fakes/yaqd_fakes/_fake_sensor.py @@ -28,7 +28,7 @@ def __init__(self, name, config, config_filepath): self._channel_generators[name] = random_walk(min_, max_) else: raise Exception(f"channel kind {kwargs['kind']} not recognized") - asyncio.get_event_loop().create_task(self._update_measurements()) + self._loop.create_task(self._update_measurements()) async def _update_measurements(self): while True: From 08cb06b16bc5d8a87aa14f636cf3a55490d2e34d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Oct 2025 20:39:01 +0000 Subject: [PATCH 3/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- yaqd-core/yaqd_core/_is_daemon.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/yaqd-core/yaqd_core/_is_daemon.py b/yaqd-core/yaqd_core/_is_daemon.py index a93615d..ad6535d 100755 --- a/yaqd-core/yaqd_core/_is_daemon.py +++ b/yaqd-core/yaqd_core/_is_daemon.py @@ -190,9 +190,7 @@ def main(cls): # Run the event loop try: - asyncio.run( - cls._main(config_filepath, config_file, args) - ) + asyncio.run(cls._main(config_filepath, config_file, args)) except asyncio.CancelledError: pass @@ -300,9 +298,11 @@ async def shutdown_all(cls, sig, loop): # are not themselves cancelled. [d.close() for d in cls._daemons] tasks = [ - t for t in asyncio.all_tasks() + t + for t in asyncio.all_tasks() if ( - t is not asyncio.current_task() + t + is not asyncio.current_task() # and "serve_forever" not in t.get_coro().__repr__() ) ] From bc4c797c28c11837f31e0a9d90fbac65b50d2e8e Mon Sep 17 00:00:00 2001 From: Daniel Kohler <11864045+ddkohler@users.noreply.github.com> Date: Mon, 20 Oct 2025 15:46:56 -0500 Subject: [PATCH 4/9] cleanup --- yaqd-core/yaqd_core/_is_daemon.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/yaqd-core/yaqd_core/_is_daemon.py b/yaqd-core/yaqd_core/_is_daemon.py index ad6535d..44e6a32 100755 --- a/yaqd-core/yaqd_core/_is_daemon.py +++ b/yaqd-core/yaqd_core/_is_daemon.py @@ -123,6 +123,7 @@ def _traits(cls) -> List[str]: @classmethod def main(cls): + """parse arguments and start the event loop""" parser = argparse.ArgumentParser() parser.add_argument( "--config", @@ -248,10 +249,8 @@ def server(daemon): server(daemon), config.get("host", ""), config.get("port", None) ) daemon._server = ser - # cls.__servers.add(asyncio.create_task()) task = asyncio.create_task(ser.serve_forever()) cls.__servers.add(task) - # Add a done callback to remove the task from the set when it completes task.add_done_callback(cls.__servers.discard) @classmethod @@ -297,21 +296,10 @@ async def shutdown_all(cls, sig, loop): # This is done after cancelling so that shutdown tasks which require the loop # are not themselves cancelled. [d.close() for d in cls._daemons] - tasks = [ - t - for t in asyncio.all_tasks() - if ( - t - is not asyncio.current_task() - # and "serve_forever" not in t.get_coro().__repr__() - ) - ] - logger.info("gathering") + tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] for task in tasks: logger.info(task.get_coro()) await asyncio.gather(*tasks, return_exceptions=True) - logger.info("gathered") - logger.info([task.get_coro() for task in asyncio.all_tasks()]) [d._save_state() for d in cls._daemons] if hasattr(signal, "SIGHUP") and sig == signal.SIGHUP: config_filepath = [d._config_filepath for d in cls._daemons][0] From f5f95e05480972d91aa2609d4dbf8234b110792a Mon Sep 17 00:00:00 2001 From: Daniel Kohler <11864045+ddkohler@users.noreply.github.com> Date: Mon, 20 Oct 2025 16:00:47 -0500 Subject: [PATCH 5/9] Update _is_daemon.py cleanup --- yaqd-core/yaqd_core/_is_daemon.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/yaqd-core/yaqd_core/_is_daemon.py b/yaqd-core/yaqd_core/_is_daemon.py index 44e6a32..0ba6cd7 100755 --- a/yaqd-core/yaqd_core/_is_daemon.py +++ b/yaqd-core/yaqd_core/_is_daemon.py @@ -197,7 +197,7 @@ def main(cls): @classmethod async def _main(cls, config_filepath, config_file, args=None): - """Parse command line arguments, start event loop tasks.""" + """Parse command line arguments, run event loop.""" loop = asyncio.get_running_loop() if sys.platform.startswith("win"): signals = () @@ -297,8 +297,6 @@ async def shutdown_all(cls, sig, loop): # are not themselves cancelled. [d.close() for d in cls._daemons] tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] - for task in tasks: - logger.info(task.get_coro()) await asyncio.gather(*tasks, return_exceptions=True) [d._save_state() for d in cls._daemons] if hasattr(signal, "SIGHUP") and sig == signal.SIGHUP: From e258cc4be87f7b51feefb44aeb419a93ceaf001f Mon Sep 17 00:00:00 2001 From: Daniel Kohler <11864045+ddkohler@users.noreply.github.com> Date: Tue, 21 Oct 2025 00:26:28 -0500 Subject: [PATCH 6/9] peek at py3.8 tests --- .github/workflows/python-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-pytest.yml b/.github/workflows/python-pytest.yml index 7cc9631..a2fec01 100644 --- a/.github/workflows/python-pytest.yml +++ b/.github/workflows/python-pytest.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} From 57cc2afc73b679b7ea3abc34d9d7999d8d6977d8 Mon Sep 17 00:00:00 2001 From: Daniel Kohler <11864045+ddkohler@users.noreply.github.com> Date: Wed, 29 Oct 2025 00:23:51 -0500 Subject: [PATCH 7/9] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 90aa4fe..248920a 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ coverage.xml # vim *.sw? +/.vscode From f4d1e5632fcf93d7884aa875c591e212de35cbc2 Mon Sep 17 00:00:00 2001 From: Daniel Kohler <11864045+ddkohler@users.noreply.github.com> Date: Thu, 6 Nov 2025 09:42:25 -0800 Subject: [PATCH 8/9] IsSensor type hints allow multidimensional channels (#82) * Update _is_sensor.py * Update _is_sensor.py * Update _is_sensor.py * Update _is_sensor.py * Update _is_sensor.py * Update _is_sensor.py * Update _is_sensor.py * Update _is_sensor.py * Update _is_sensor.py * Update _is_sensor.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * propagate to HasMeasureTrigger * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update _is_sensor.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove MeasureType * Update CHANGELOG.md * remove redundant actions * Update README.md * expand test py versions * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update pyproject.toml * initiate tests again * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * try python 3.13 curious if the freze issue still persists * Update _is_daemon.py * Update _is_daemon.py * Update _is_daemon.py * Update pyproject.toml * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/python-pytest.yml | 5 +++-- .github/workflows/run-entry-points.yml | 5 +++-- README.md | 2 +- yaqd-core/CHANGELOG.md | 3 +++ yaqd-core/pyproject.toml | 5 +++-- yaqd-core/yaqd_core/_has_measure_trigger.py | 11 ++++++----- yaqd-core/yaqd_core/_is_daemon.py | 13 +++++++++++-- yaqd-core/yaqd_core/_is_sensor.py | 18 +++++++++--------- 8 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/workflows/python-pytest.yml b/.github/workflows/python-pytest.yml index e4d7945..7cc9631 100644 --- a/.github/workflows/python-pytest.yml +++ b/.github/workflows/python-pytest.yml @@ -1,15 +1,16 @@ name: pytest on: - push: pull_request: + types: [opened, reopened] + push: jobs: build: strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/run-entry-points.yml b/.github/workflows/run-entry-points.yml index f25ba88..2ef96e8 100644 --- a/.github/workflows/run-entry-points.yml +++ b/.github/workflows/run-entry-points.yml @@ -1,15 +1,16 @@ name: run entry points on: - push: pull_request: + types: [opened, reopened] + push: jobs: build: strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} diff --git a/README.md b/README.md index 0795ef9..04a55c5 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Repository for yaq core python packages: Each of these projects is distributed as a separate package. -These core packages are kept in one repository primarily such that, when needed, changes can be made simulatiously to multiple packages whithout breaking tests. +These core packages are kept in one repository primarily such that, when needed, changes can be made simultaneously to multiple packages without breaking tests. Repository maintainers: - [Kyle Sunden](https://github.com/ksunden) diff --git a/yaqd-core/CHANGELOG.md b/yaqd-core/CHANGELOG.md index ea5dabb..5f8d640 100644 --- a/yaqd-core/CHANGELOG.md +++ b/yaqd-core/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). ## [Unreleased] +### Fixed +- type hints for IsSensor attributes are appropriate for _n_-dimensional data + ## [2023.11.0] ### Fixed diff --git a/yaqd-core/pyproject.toml b/yaqd-core/pyproject.toml index 4d2fac2..8bdd546 100644 --- a/yaqd-core/pyproject.toml +++ b/yaqd-core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "yaqd-core" -author = [{name="yaq developers"}] +authors = [{name="yaq developers"}] requires-python = ">=3.7" dependencies = ["platformdirs", "tomli", "tomli-w", "fastavro>=1.4.0"] readme="README.md" @@ -20,6 +20,7 @@ classifiers=[ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering", ] @@ -37,7 +38,7 @@ path = "yaqd_core/__version__.py" [tool.black] line-length = 99 -target-version = ['py37', 'py38'] +target-version = ['py311', "py312"] include = '\.pyi?$' exclude = ''' /( diff --git a/yaqd-core/yaqd_core/_has_measure_trigger.py b/yaqd-core/yaqd_core/_has_measure_trigger.py index a3f3447..ed25962 100644 --- a/yaqd-core/yaqd_core/_has_measure_trigger.py +++ b/yaqd-core/yaqd_core/_has_measure_trigger.py @@ -1,29 +1,30 @@ +from __future__ import annotations + __all__ = ["HasMeasureTrigger"] import asyncio import pathlib -from typing import Dict, Any, Optional +from typing import Any from abc import ABC, abstractmethod from yaqd_core import IsSensor, IsDaemon -from ._is_sensor import MeasureType class HasMeasureTrigger(IsSensor, IsDaemon, ABC): def __init__( - self, name: str, config: Dict[str, Any], config_filepath: pathlib.Path + self, name: str, config: dict[str, Any], config_filepath: pathlib.Path ): super().__init__(name, config, config_filepath) self._looping = False if self._config["loop_at_startup"]: self.measure(loop=True) - def get_measured(self) -> MeasureType: + def get_measured(self) -> dict: return super().get_measured() @abstractmethod - async def _measure(self) -> MeasureType: + async def _measure(self) -> dict: """Do measurement, filling _measured dictionary. Returns dictionary with keys channel names, values numbers or arrays. diff --git a/yaqd-core/yaqd_core/_is_daemon.py b/yaqd-core/yaqd_core/_is_daemon.py index c6f03e1..c2940fe 100755 --- a/yaqd-core/yaqd_core/_is_daemon.py +++ b/yaqd-core/yaqd_core/_is_daemon.py @@ -87,7 +87,7 @@ def __init__( self._busy_sig = asyncio.Event() self._not_busy_sig = asyncio.Event() - self._loop = asyncio.get_event_loop() + self._loop = asyncio.get_running_loop() try: self._state_filepath.parent.mkdir(parents=True, exist_ok=True) @@ -297,7 +297,16 @@ async def shutdown_all(cls, sig, loop): # This is done after cancelling so that shutdown tasks which require the loop # are not themselves cancelled. [d.close() for d in cls._daemons] - tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] + tasks = [ + t + for t in asyncio.all_tasks() + if ( + t is not asyncio.current_task() + and "serve_forever" not in t.get_coro().__repr__() + ) + ] + for task in tasks: + logger.info(task.get_coro()) await asyncio.gather(*tasks, return_exceptions=True) [d._save_state() for d in cls._daemons] if hasattr(signal, "SIGHUP") and sig == signal.SIGHUP: diff --git a/yaqd-core/yaqd_core/_is_sensor.py b/yaqd-core/yaqd_core/_is_sensor.py index 735269b..1e050f9 100644 --- a/yaqd-core/yaqd_core/_is_sensor.py +++ b/yaqd-core/yaqd_core/_is_sensor.py @@ -1,24 +1,24 @@ +from __future__ import annotations + __all__ = ["IsSensor"] import asyncio import pathlib -from typing import Dict, Any, Union, Tuple, List +from typing import Any import yaqd_core -MeasureType = Dict[str, Union[float]] - class IsSensor(yaqd_core.IsDaemon): def __init__( - self, name: str, config: Dict[str, Any], config_filepath: pathlib.Path + self, name: str, config: dict[str, Any], config_filepath: pathlib.Path ): super().__init__(name, config, config_filepath) - self._measured: MeasureType = dict() # values must be numbers or arrays - self._channel_names: List[str] = [] - self._channel_units: Dict[str, str] = dict() - self._channel_shapes: Dict[str, Tuple[int]] = dict() + self._measured: dict = dict() # values must be numbers or arrays + self._channel_names: list[str] = [] + self._channel_units: dict[str, str] = dict() + self._channel_shapes: dict[str, tuple[int, ...]] = dict() self._measurement_id = 0 self._measured["measurement_id"] = self._measurement_id @@ -37,7 +37,7 @@ def get_channel_units(self): """Get channel units.""" return self._channel_units - def get_measured(self) -> MeasureType: + def get_measured(self) -> dict: assert "measurement_id" in self._measured return self._measured From 41be3369be14039060c089ae8b85aa1d1cccc72d Mon Sep 17 00:00:00 2001 From: Daniel Kohler <11864045+ddkohler@users.noreply.github.com> Date: Thu, 6 Nov 2025 11:55:19 -0600 Subject: [PATCH 9/9] Update _is_daemon.py --- yaqd-core/yaqd_core/_is_daemon.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/yaqd-core/yaqd_core/_is_daemon.py b/yaqd-core/yaqd_core/_is_daemon.py index b8beb6a..0ba6cd7 100755 --- a/yaqd-core/yaqd_core/_is_daemon.py +++ b/yaqd-core/yaqd_core/_is_daemon.py @@ -296,16 +296,7 @@ async def shutdown_all(cls, sig, loop): # This is done after cancelling so that shutdown tasks which require the loop # are not themselves cancelled. [d.close() for d in cls._daemons] - tasks = [ - t - for t in asyncio.all_tasks() - if ( - t is not asyncio.current_task() - and "serve_forever" not in t.get_coro().__repr__() - ) - ] - for task in tasks: - logger.info(task.get_coro()) + tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] await asyncio.gather(*tasks, return_exceptions=True) [d._save_state() for d in cls._daemons] if hasattr(signal, "SIGHUP") and sig == signal.SIGHUP: