diff --git a/mypi.ini b/mypi.ini index f07db77a..738b3d27 100644 --- a/mypi.ini +++ b/mypi.ini @@ -36,9 +36,6 @@ disallow_untyped_defs = True check_untyped_defs = True disallow_untyped_defs = True -[mypy-aiomemoizettl.*] -ignore_missing_imports = True - [mypy-dictdiffer.*] ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index fad862e3..1758b796 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,10 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] dependencies = [ "aiohttp>=3", - "aiomemoizettl", "arrow>=1.0", "cryptography>=2.6.1", "dictdiffer", diff --git a/src/scriptworker/cot/verify.py b/src/scriptworker/cot/verify.py index bf701e51..0aaddf92 100644 --- a/src/scriptworker/cot/verify.py +++ b/src/scriptworker/cot/verify.py @@ -2133,7 +2133,7 @@ def verify_cot_cmdln(args=None, event_loop=None): level = logging.DEBUG if opts.verbose else logging.INFO log.setLevel(level) logging.basicConfig(level=level) - event_loop = event_loop or asyncio.get_event_loop() + event_loop = event_loop or asyncio.new_event_loop() if not opts.cleanup: log.info("Artifacts will be in {}".format(tmp)) try: diff --git a/src/scriptworker/github.py b/src/scriptworker/github.py index 2d002371..baf2e96a 100644 --- a/src/scriptworker/github.py +++ b/src/scriptworker/github.py @@ -1,9 +1,9 @@ """GitHub helper functions.""" +import asyncio import logging import re -from aiomemoizettl import memoize_ttl from github3 import GitHub from github3.exceptions import GitHubException @@ -136,24 +136,36 @@ async def has_commit_landed_on_repository(self, context, revision): return html_text != "" -# TODO Use memoize_ttl() as decorator once https://github.com/michalc/aiomemoizettl/issues/2 is done -async def _fetch_github_branch_commits_data_helper(context, repo_html_url, revision): - url = "/".join((repo_html_url.rstrip("/"), "branch_commits", revision)) - log.info('Cache does not exist for URL "{}" (in this context), fetching it...'.format(url)) - html_text = await retry_request(context, url) - return html_text.strip() +_BRANCH_COMMITS_CACHE_TTL_IN_SECONDS = 10 * 60 # 10 minutes +_BRANCH_COMMITS_CACHE = {} -# XXX memoize_ttl() uses all function parameters to create a key that stores its cache. -# This means new contexts cannot use the memoized value, even though they're calling the same -# repo and revision. jlorenzo tried to take the context out of the memoize_ttl() call, but -# whenever the cache is invalidated, request() doesn't work anymore because the session carried -# by the context has been long closed. -# Therefore, the defined TTL has 2 purposes: -# a. it memoizes calls for the time of a single cot_verify() run -# b. it clears the cache automatically, so we don't have to manually invalidate it. -_BRANCH_COMMITS_CACHE_TTL_IN_SECONDS = 10 * 60 # 10 minutes -_fetch_github_branch_commits_data = memoize_ttl(_fetch_github_branch_commits_data_helper, get_ttl=lambda _: _BRANCH_COMMITS_CACHE_TTL_IN_SECONDS) +async def _fetch_github_branch_commits_data(context, repo_html_url, revision): + cache_key = (id(context), repo_html_url, revision) + + if cache_key in _BRANCH_COMMITS_CACHE: + return await _BRANCH_COMMITS_CACHE[cache_key] + + future = asyncio.get_running_loop().create_future() + _BRANCH_COMMITS_CACHE[cache_key] = future + + try: + url = "/".join((repo_html_url.rstrip("/"), "branch_commits", revision)) + html_text = await retry_request(context, url) + result = html_text.strip() + future.set_result(result) + asyncio.get_running_loop().call_later( + _BRANCH_COMMITS_CACHE_TTL_IN_SECONDS, + _BRANCH_COMMITS_CACHE.pop, + cache_key, + None, + ) + except BaseException as e: + _BRANCH_COMMITS_CACHE.pop(cache_key, None) + future.set_exception(e) + raise + + return result def is_github_url(url): diff --git a/taskcluster/kinds/docker-image/kind.yml b/taskcluster/kinds/docker-image/kind.yml index 4476adfb..c8334661 100644 --- a/taskcluster/kinds/docker-image/kind.yml +++ b/taskcluster/kinds/docker-image/kind.yml @@ -12,9 +12,13 @@ transforms: task-defaults: args: - UV_VERSION: "0.7.9" + UV_VERSION: "0.9.16" tasks: + python3.14: + definition: python + args: + PYTHON_VERSION: "3.14" python3.13: definition: python args: diff --git a/taskcluster/kinds/tox/kind.yml b/taskcluster/kinds/tox/kind.yml index 4ac60dac..f806d361 100644 --- a/taskcluster/kinds/tox/kind.yml +++ b/taskcluster/kinds/tox/kind.yml @@ -51,6 +51,11 @@ tasks: targets: py313,check env: NO_TESTS_OVER_WIRE: "1" + py314: + python-version: "3.14" + targets: py314,check + env: + NO_TESTS_OVER_WIRE: "1" py311-cot: python-version: "3.11" targets: py311-cot @@ -72,3 +77,10 @@ tasks: NO_CREDENTIALS_TESTS: "1" scopes: - secrets:get:repo:github.com/mozilla-releng/scriptworker:github + py314-cot: + python-version: "3.14" + targets: py314-cot + env: + NO_CREDENTIALS_TESTS: "1" + scopes: + - secrets:get:repo:github.com/mozilla-releng/scriptworker:github diff --git a/tests/data/long_running.py b/tests/data/long_running.py index 0f8954ee..43a1e6e7 100644 --- a/tests/data/long_running.py +++ b/tests/data/long_running.py @@ -23,7 +23,7 @@ def launch_second_instances(): if not os.path.exists(temp_dir): os.makedirs(temp_dir) job1 = subprocess.Popen([sys.executable, __file__, os.path.join(temp_dir, "one"), os.path.join(temp_dir, "two"), os.path.join(temp_dir, "three")]) - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() job2 = asyncio.create_subprocess_exec( sys.executable, __file__, os.path.join(temp_dir, "four"), os.path.join(temp_dir, "five"), os.path.join(temp_dir, "six") ) @@ -39,7 +39,7 @@ async def write_file1(path): def write_files(): - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() task = write_file1(sys.argv[1]) subprocess.Popen(["bash", BASH_SCRIPT, sys.argv[2]]) subprocess.Popen("bash {} {}".format(BASH_SCRIPT, sys.argv[3]), shell=True) diff --git a/tests/test_github.py b/tests/test_github.py index 3e3e8948..5a51677e 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -8,20 +8,12 @@ from scriptworker import github from scriptworker.exceptions import ConfigError -_CACHE = {} - -@pytest.fixture(scope="session", autouse=True) -async def mock_memoized_func(): - # Memoize_ttl causes an issue with pytest-asyncio, so we copy the behavior to an in-memory cache - async def fetch(*args, **kwargs): - key = (args, tuple(kwargs.items())) - if key not in _CACHE: - _CACHE[key] = await github._fetch_github_branch_commits_data_helper(*args, **kwargs) - return _CACHE[key] - - with patch("scriptworker.github._fetch_github_branch_commits_data", fetch): - yield +@pytest.fixture(autouse=True) +def clear_github_cache(): + github._BRANCH_COMMITS_CACHE.clear() + yield + github._BRANCH_COMMITS_CACHE.clear() @pytest.fixture(scope="function") diff --git a/tests/test_production.py b/tests/test_production.py index a32cc8bc..e658194c 100644 --- a/tests/test_production.py +++ b/tests/test_production.py @@ -6,7 +6,6 @@ import os import re import tempfile -from unittest.mock import patch import pytest from asyncio_extras.contextmanager import async_contextmanager @@ -14,7 +13,6 @@ import scriptworker.log as swlog import scriptworker.utils as utils -from scriptworker import github from scriptworker.config import apply_product_config, get_unfrozen_copy, read_worker_creds from scriptworker.constants import DEFAULT_CONFIG from scriptworker.context import Context @@ -24,20 +22,6 @@ # constants helpers and fixtures {{{1 pytestmark = [pytest.mark.skipif(os.environ.get("NO_TESTS_OVER_WIRE"), reason="NO_TESTS_OVER_WIRE: skipping production CoT verification test")] -_CACHE = {} - - -@pytest.fixture(scope="session", autouse=True) -async def mock_memoized_func(): - # Memoize_ttl causes an issue with pytest-asyncio, so we copy the behavior to an in-memory cache - async def fetch(*args, **kwargs): - key = (args, tuple(kwargs.items())) - if key not in _CACHE: - _CACHE[key] = await github._fetch_github_branch_commits_data_helper(*args, **kwargs) - return _CACHE[key] - - with patch("scriptworker.github._fetch_github_branch_commits_data", fetch): - yield def read_integration_creds(): diff --git a/tox.ini b/tox.ini index 3d4b324b..3798ef8c 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,10 @@ commands= commands= py.test -k test_verify_production_cot --random-order-bucket=none +[testenv:py314-cot] +commands= + py.test -k test_verify_production_cot --random-order-bucket=none + [testenv:check] skip_install = true commands =