Add support for python 3.14

mypi.ini

@@ -36,9 +36,6 @@ disallow_untyped_defs = True
 check_untyped_defs = True
 disallow_untyped_defs = True
 
-[mypy-aiomemoizettl.*]
-ignore_missing_imports = True
-
 [mypy-dictdiffer.*]
 ignore_missing_imports = True
 

pyproject.toml

@@ -13,10 +13,10 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
 ]
 dependencies = [
     "aiohttp>=3",
-    "aiomemoizettl",
     "arrow>=1.0",
     "cryptography>=2.6.1",
     "dictdiffer",

src/scriptworker/cot/verify.py

@@ -2133,7 +2133,7 @@ def verify_cot_cmdln(args=None, event_loop=None):
     level = logging.DEBUG if opts.verbose else logging.INFO
     log.setLevel(level)
     logging.basicConfig(level=level)
-    event_loop = event_loop or asyncio.get_event_loop()
+    event_loop = event_loop or asyncio.new_event_loop()
     if not opts.cleanup:
         log.info("Artifacts will be in {}".format(tmp))
     try:

src/scriptworker/github.py

@@ -1,9 +1,9 @@
 """GitHub helper functions."""
 
+import asyncio
 import logging
 import re
 
-from aiomemoizettl import memoize_ttl
 from github3 import GitHub
 from github3.exceptions import GitHubException
 
@@ -136,24 +136,36 @@ async def has_commit_landed_on_repository(self, context, revision):
         return html_text != ""
 
 
-# TODO Use memoize_ttl() as decorator once https://github.com/michalc/aiomemoizettl/issues/2 is done
-async def _fetch_github_branch_commits_data_helper(context, repo_html_url, revision):
-    url = "/".join((repo_html_url.rstrip("/"), "branch_commits", revision))
-    log.info('Cache does not exist for URL "{}" (in this context), fetching it...'.format(url))
-    html_text = await retry_request(context, url)
-    return html_text.strip()
+_BRANCH_COMMITS_CACHE_TTL_IN_SECONDS = 10 * 60  # 10 minutes
+_BRANCH_COMMITS_CACHE = {}
 
 
-# XXX memoize_ttl() uses all function parameters to create a key that stores its cache.
-# This means new contexts cannot use the memoized value, even though they're calling the same
-# repo and revision. jlorenzo tried to take the context out of the memoize_ttl() call, but
-# whenever the cache is invalidated, request() doesn't work anymore because the session carried
-# by the context has been long closed.
-# Therefore, the defined TTL has 2 purposes:
-#  a. it memoizes calls for the time of a single cot_verify() run
-#  b. it clears the cache automatically, so we don't have to manually invalidate it.
-_BRANCH_COMMITS_CACHE_TTL_IN_SECONDS = 10 * 60  # 10 minutes
-_fetch_github_branch_commits_data = memoize_ttl(_fetch_github_branch_commits_data_helper, get_ttl=lambda _: _BRANCH_COMMITS_CACHE_TTL_IN_SECONDS)
+async def _fetch_github_branch_commits_data(context, repo_html_url, revision):
+    cache_key = (id(context), repo_html_url, revision)
+
+    if cache_key in _BRANCH_COMMITS_CACHE:
+        return await _BRANCH_COMMITS_CACHE[cache_key]
+
+    future = asyncio.get_running_loop().create_future()
+    _BRANCH_COMMITS_CACHE[cache_key] = future
+
+    try:
+        url = "/".join((repo_html_url.rstrip("/"), "branch_commits", revision))
+        html_text = await retry_request(context, url)
+        result = html_text.strip()
+        future.set_result(result)
+        asyncio.get_running_loop().call_later(
+            _BRANCH_COMMITS_CACHE_TTL_IN_SECONDS,
+            _BRANCH_COMMITS_CACHE.pop,
+            cache_key,
+            None,
+        )
+    except BaseException as e:
+        _BRANCH_COMMITS_CACHE.pop(cache_key, None)
+        future.set_exception(e)
+        raise
+
+    return result
 
 
 def is_github_url(url):

taskcluster/kinds/docker-image/kind.yml

@@ -12,9 +12,13 @@ transforms:
 
 task-defaults:
   args:
-    UV_VERSION: "0.7.9"
+    UV_VERSION: "0.9.16"
 
 tasks:
+    python3.14:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.14"
     python3.13:
         definition: python
         args:

taskcluster/kinds/tox/kind.yml

@@ -51,6 +51,11 @@ tasks:
         targets: py313,check
         env:
             NO_TESTS_OVER_WIRE: "1"
+    py314:
+        python-version: "3.14"
+        targets: py314,check
+        env:
+            NO_TESTS_OVER_WIRE: "1"
     py311-cot:
         python-version: "3.11"
         targets: py311-cot
@@ -72,3 +77,10 @@ tasks:
             NO_CREDENTIALS_TESTS: "1"
         scopes:
             - secrets:get:repo:github.com/mozilla-releng/scriptworker:github
+    py314-cot:
+        python-version: "3.14"
+        targets: py314-cot
+        env:
+            NO_CREDENTIALS_TESTS: "1"
+        scopes:
+            - secrets:get:repo:github.com/mozilla-releng/scriptworker:github

tests/data/long_running.py

@@ -23,7 +23,7 @@ def launch_second_instances():
     if not os.path.exists(temp_dir):
         os.makedirs(temp_dir)
     job1 = subprocess.Popen([sys.executable, __file__, os.path.join(temp_dir, "one"), os.path.join(temp_dir, "two"), os.path.join(temp_dir, "three")])
-    loop = asyncio.get_event_loop()
+    loop = asyncio.new_event_loop()
     job2 = asyncio.create_subprocess_exec(
         sys.executable, __file__, os.path.join(temp_dir, "four"), os.path.join(temp_dir, "five"), os.path.join(temp_dir, "six")
     )
@@ -39,7 +39,7 @@ async def write_file1(path):
 
 
 def write_files():
-    loop = asyncio.get_event_loop()
+    loop = asyncio.new_event_loop()
     task = write_file1(sys.argv[1])
     subprocess.Popen(["bash", BASH_SCRIPT, sys.argv[2]])
     subprocess.Popen("bash {} {}".format(BASH_SCRIPT, sys.argv[3]), shell=True)

tests/test_github.py

@@ -8,20 +8,12 @@
 from scriptworker import github
 from scriptworker.exceptions import ConfigError
 
-_CACHE = {}
 
-
-@pytest.fixture(scope="session", autouse=True)
-async def mock_memoized_func():
-    # Memoize_ttl causes an issue with pytest-asyncio, so we copy the behavior to an in-memory cache
-    async def fetch(*args, **kwargs):
-        key = (args, tuple(kwargs.items()))
-        if key not in _CACHE:
-            _CACHE[key] = await github._fetch_github_branch_commits_data_helper(*args, **kwargs)
-        return _CACHE[key]
-
-    with patch("scriptworker.github._fetch_github_branch_commits_data", fetch):
-        yield
+@pytest.fixture(autouse=True)
+def clear_github_cache():
+    github._BRANCH_COMMITS_CACHE.clear()
+    yield
+    github._BRANCH_COMMITS_CACHE.clear()
 
 
 @pytest.fixture(scope="function")

tests/test_production.py

@@ -6,15 +6,13 @@
 import os
 import re
 import tempfile
-from unittest.mock import patch
 
 import pytest
 from asyncio_extras.contextmanager import async_contextmanager
 from taskcluster.aio import Index, Queue, Secrets
 
 import scriptworker.log as swlog
 import scriptworker.utils as utils
-from scriptworker import github
 from scriptworker.config import apply_product_config, get_unfrozen_copy, read_worker_creds
 from scriptworker.constants import DEFAULT_CONFIG
 from scriptworker.context import Context
@@ -24,20 +22,6 @@
 
 # constants helpers and fixtures {{{1
 pytestmark = [pytest.mark.skipif(os.environ.get("NO_TESTS_OVER_WIRE"), reason="NO_TESTS_OVER_WIRE: skipping production CoT verification test")]
-_CACHE = {}
-
-
-@pytest.fixture(scope="session", autouse=True)
-async def mock_memoized_func():
-    # Memoize_ttl causes an issue with pytest-asyncio, so we copy the behavior to an in-memory cache
-    async def fetch(*args, **kwargs):
-        key = (args, tuple(kwargs.items()))
-        if key not in _CACHE:
-            _CACHE[key] = await github._fetch_github_branch_commits_data_helper(*args, **kwargs)
-        return _CACHE[key]
-
-    with patch("scriptworker.github._fetch_github_branch_commits_data", fetch):
-        yield
 
 
 def read_integration_creds():

tox.ini

@@ -48,6 +48,10 @@ commands=
 commands=
     py.test -k test_verify_production_cot --random-order-bucket=none
 
+[testenv:py314-cot]
+commands=
+    py.test -k test_verify_production_cot --random-order-bucket=none
+
 [testenv:check]
 skip_install = true
 commands =