From ab199e5f245f939d2751f26b522febed608b9289 Mon Sep 17 00:00:00 2001 From: Bastien Orivel Date: Mon, 8 Dec 2025 14:00:26 +0100 Subject: [PATCH 1/4] Replace aiomemoizettl by an inline cache I'm not too fond of reinventing the wheel but we only have one use of this and I don't feel like adding yet another dependency for something that's 20 lines of code... This is necessary to get scriptworker to work with python3.14 --- mypi.ini | 3 --- pyproject.toml | 1 - src/scriptworker/github.py | 46 ++++++++++++++++++++++++-------------- tests/test_github.py | 18 +++++---------- tests/test_production.py | 16 ------------- 5 files changed, 34 insertions(+), 50 deletions(-) diff --git a/mypi.ini b/mypi.ini index f07db77a7..738b3d277 100644 --- a/mypi.ini +++ b/mypi.ini @@ -36,9 +36,6 @@ disallow_untyped_defs = True check_untyped_defs = True disallow_untyped_defs = True -[mypy-aiomemoizettl.*] -ignore_missing_imports = True - [mypy-dictdiffer.*] ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index fad862e3b..58cb0126c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ classifiers = [ ] dependencies = [ "aiohttp>=3", - "aiomemoizettl", "arrow>=1.0", "cryptography>=2.6.1", "dictdiffer", diff --git a/src/scriptworker/github.py b/src/scriptworker/github.py index 2d0023717..baf2e96ae 100644 --- a/src/scriptworker/github.py +++ b/src/scriptworker/github.py @@ -1,9 +1,9 @@ """GitHub helper functions.""" +import asyncio import logging import re -from aiomemoizettl import memoize_ttl from github3 import GitHub from github3.exceptions import GitHubException @@ -136,24 +136,36 @@ async def has_commit_landed_on_repository(self, context, revision): return html_text != "" -# TODO Use memoize_ttl() as decorator once https://github.com/michalc/aiomemoizettl/issues/2 is done -async def _fetch_github_branch_commits_data_helper(context, repo_html_url, revision): - url = "/".join((repo_html_url.rstrip("/"), "branch_commits", revision)) - log.info('Cache does not exist for URL "{}" (in this context), fetching it...'.format(url)) - html_text = await retry_request(context, url) - return html_text.strip() +_BRANCH_COMMITS_CACHE_TTL_IN_SECONDS = 10 * 60 # 10 minutes +_BRANCH_COMMITS_CACHE = {} -# XXX memoize_ttl() uses all function parameters to create a key that stores its cache. -# This means new contexts cannot use the memoized value, even though they're calling the same -# repo and revision. jlorenzo tried to take the context out of the memoize_ttl() call, but -# whenever the cache is invalidated, request() doesn't work anymore because the session carried -# by the context has been long closed. -# Therefore, the defined TTL has 2 purposes: -# a. it memoizes calls for the time of a single cot_verify() run -# b. it clears the cache automatically, so we don't have to manually invalidate it. -_BRANCH_COMMITS_CACHE_TTL_IN_SECONDS = 10 * 60 # 10 minutes -_fetch_github_branch_commits_data = memoize_ttl(_fetch_github_branch_commits_data_helper, get_ttl=lambda _: _BRANCH_COMMITS_CACHE_TTL_IN_SECONDS) +async def _fetch_github_branch_commits_data(context, repo_html_url, revision): + cache_key = (id(context), repo_html_url, revision) + + if cache_key in _BRANCH_COMMITS_CACHE: + return await _BRANCH_COMMITS_CACHE[cache_key] + + future = asyncio.get_running_loop().create_future() + _BRANCH_COMMITS_CACHE[cache_key] = future + + try: + url = "/".join((repo_html_url.rstrip("/"), "branch_commits", revision)) + html_text = await retry_request(context, url) + result = html_text.strip() + future.set_result(result) + asyncio.get_running_loop().call_later( + _BRANCH_COMMITS_CACHE_TTL_IN_SECONDS, + _BRANCH_COMMITS_CACHE.pop, + cache_key, + None, + ) + except BaseException as e: + _BRANCH_COMMITS_CACHE.pop(cache_key, None) + future.set_exception(e) + raise + + return result def is_github_url(url): diff --git a/tests/test_github.py b/tests/test_github.py index 3e3e8948b..5a51677ea 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -8,20 +8,12 @@ from scriptworker import github from scriptworker.exceptions import ConfigError -_CACHE = {} - -@pytest.fixture(scope="session", autouse=True) -async def mock_memoized_func(): - # Memoize_ttl causes an issue with pytest-asyncio, so we copy the behavior to an in-memory cache - async def fetch(*args, **kwargs): - key = (args, tuple(kwargs.items())) - if key not in _CACHE: - _CACHE[key] = await github._fetch_github_branch_commits_data_helper(*args, **kwargs) - return _CACHE[key] - - with patch("scriptworker.github._fetch_github_branch_commits_data", fetch): - yield +@pytest.fixture(autouse=True) +def clear_github_cache(): + github._BRANCH_COMMITS_CACHE.clear() + yield + github._BRANCH_COMMITS_CACHE.clear() @pytest.fixture(scope="function") diff --git a/tests/test_production.py b/tests/test_production.py index a32cc8bc7..e658194c5 100644 --- a/tests/test_production.py +++ b/tests/test_production.py @@ -6,7 +6,6 @@ import os import re import tempfile -from unittest.mock import patch import pytest from asyncio_extras.contextmanager import async_contextmanager @@ -14,7 +13,6 @@ import scriptworker.log as swlog import scriptworker.utils as utils -from scriptworker import github from scriptworker.config import apply_product_config, get_unfrozen_copy, read_worker_creds from scriptworker.constants import DEFAULT_CONFIG from scriptworker.context import Context @@ -24,20 +22,6 @@ # constants helpers and fixtures {{{1 pytestmark = [pytest.mark.skipif(os.environ.get("NO_TESTS_OVER_WIRE"), reason="NO_TESTS_OVER_WIRE: skipping production CoT verification test")] -_CACHE = {} - - -@pytest.fixture(scope="session", autouse=True) -async def mock_memoized_func(): - # Memoize_ttl causes an issue with pytest-asyncio, so we copy the behavior to an in-memory cache - async def fetch(*args, **kwargs): - key = (args, tuple(kwargs.items())) - if key not in _CACHE: - _CACHE[key] = await github._fetch_github_branch_commits_data_helper(*args, **kwargs) - return _CACHE[key] - - with patch("scriptworker.github._fetch_github_branch_commits_data", fetch): - yield def read_integration_creds(): From b011073bd1946afbc1632bb41ed3511aa51b46d7 Mon Sep 17 00:00:00 2001 From: Bastien Orivel Date: Mon, 8 Dec 2025 14:03:43 +0100 Subject: [PATCH 2/4] Don't use `get_event_loop` when there's no event loop running This fixes python 3.14 compatibility. `get_event_loop` used to create an event loop if there wasn't any but 3.14 changed that behavior so we have to explicitely create one now --- src/scriptworker/cot/verify.py | 2 +- tests/data/long_running.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scriptworker/cot/verify.py b/src/scriptworker/cot/verify.py index bf701e518..0aaddf92c 100644 --- a/src/scriptworker/cot/verify.py +++ b/src/scriptworker/cot/verify.py @@ -2133,7 +2133,7 @@ def verify_cot_cmdln(args=None, event_loop=None): level = logging.DEBUG if opts.verbose else logging.INFO log.setLevel(level) logging.basicConfig(level=level) - event_loop = event_loop or asyncio.get_event_loop() + event_loop = event_loop or asyncio.new_event_loop() if not opts.cleanup: log.info("Artifacts will be in {}".format(tmp)) try: diff --git a/tests/data/long_running.py b/tests/data/long_running.py index 0f8954ee9..43a1e6e77 100644 --- a/tests/data/long_running.py +++ b/tests/data/long_running.py @@ -23,7 +23,7 @@ def launch_second_instances(): if not os.path.exists(temp_dir): os.makedirs(temp_dir) job1 = subprocess.Popen([sys.executable, __file__, os.path.join(temp_dir, "one"), os.path.join(temp_dir, "two"), os.path.join(temp_dir, "three")]) - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() job2 = asyncio.create_subprocess_exec( sys.executable, __file__, os.path.join(temp_dir, "four"), os.path.join(temp_dir, "five"), os.path.join(temp_dir, "six") ) @@ -39,7 +39,7 @@ async def write_file1(path): def write_files(): - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() task = write_file1(sys.argv[1]) subprocess.Popen(["bash", BASH_SCRIPT, sys.argv[2]]) subprocess.Popen("bash {} {}".format(BASH_SCRIPT, sys.argv[3]), shell=True) From 81f57966c00fa35ce0407ce7954a00bdc783c8c3 Mon Sep 17 00:00:00 2001 From: Bastien Orivel Date: Mon, 8 Dec 2025 14:09:19 +0100 Subject: [PATCH 3/4] Add support for python 3.14 --- pyproject.toml | 1 + taskcluster/kinds/docker-image/kind.yml | 4 ++++ taskcluster/kinds/tox/kind.yml | 12 ++++++++++++ tox.ini | 4 ++++ 4 files changed, 21 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 58cb0126c..1758b7966 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] dependencies = [ "aiohttp>=3", diff --git a/taskcluster/kinds/docker-image/kind.yml b/taskcluster/kinds/docker-image/kind.yml index 4476adfbe..81698270e 100644 --- a/taskcluster/kinds/docker-image/kind.yml +++ b/taskcluster/kinds/docker-image/kind.yml @@ -15,6 +15,10 @@ task-defaults: UV_VERSION: "0.7.9" tasks: + python3.14: + definition: python + args: + PYTHON_VERSION: "3.14" python3.13: definition: python args: diff --git a/taskcluster/kinds/tox/kind.yml b/taskcluster/kinds/tox/kind.yml index 4ac60dacf..f806d3613 100644 --- a/taskcluster/kinds/tox/kind.yml +++ b/taskcluster/kinds/tox/kind.yml @@ -51,6 +51,11 @@ tasks: targets: py313,check env: NO_TESTS_OVER_WIRE: "1" + py314: + python-version: "3.14" + targets: py314,check + env: + NO_TESTS_OVER_WIRE: "1" py311-cot: python-version: "3.11" targets: py311-cot @@ -72,3 +77,10 @@ tasks: NO_CREDENTIALS_TESTS: "1" scopes: - secrets:get:repo:github.com/mozilla-releng/scriptworker:github + py314-cot: + python-version: "3.14" + targets: py314-cot + env: + NO_CREDENTIALS_TESTS: "1" + scopes: + - secrets:get:repo:github.com/mozilla-releng/scriptworker:github diff --git a/tox.ini b/tox.ini index 3d4b324bc..3798ef8c3 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,10 @@ commands= commands= py.test -k test_verify_production_cot --random-order-bucket=none +[testenv:py314-cot] +commands= + py.test -k test_verify_production_cot --random-order-bucket=none + [testenv:check] skip_install = true commands = From 028bcbdd65152503af0b5fa25613d3335d50114d Mon Sep 17 00:00:00 2001 From: Bastien Orivel Date: Mon, 8 Dec 2025 15:31:54 +0100 Subject: [PATCH 4/4] Update uv to 0.9.16 There's no `0.7.9-python3.14-bookworm-slim` image but there is a `0.9.16-python3.14-bookworm-slim` one. --- taskcluster/kinds/docker-image/kind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/kinds/docker-image/kind.yml b/taskcluster/kinds/docker-image/kind.yml index 81698270e..c83346613 100644 --- a/taskcluster/kinds/docker-image/kind.yml +++ b/taskcluster/kinds/docker-image/kind.yml @@ -12,7 +12,7 @@ transforms: task-defaults: args: - UV_VERSION: "0.7.9" + UV_VERSION: "0.9.16" tasks: python3.14: