From 45933fe6c311f52e7ae2a8955ffc18c55bd56620 Mon Sep 17 00:00:00 2001 From: jonathan keinan Date: Tue, 21 Oct 2025 18:13:22 +0300 Subject: [PATCH 01/11] initial commit --- RLTest/__main__.py | 58 +++++++++++++++++++++++++++++++++++++++------- RLTest/utils.py | 6 +++++ 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/RLTest/__main__.py b/RLTest/__main__.py index 8c0754b..1578e8f 100644 --- a/RLTest/__main__.py +++ b/RLTest/__main__.py @@ -15,7 +15,7 @@ from multiprocessing import Process, Queue, set_start_method from RLTest.env import Env, TestAssertionFailure, Defaults -from RLTest.utils import Colors, fix_modules, fix_modulesArgs +from RLTest.utils import Colors, fix_modules, fix_modulesArgs, is_github_actions from RLTest.loader import TestLoader from RLTest.Enterprise import binaryrepo from RLTest import debuggers @@ -553,6 +553,9 @@ def __init__(self): self.testsFailed = {} self.currEnv = None self.loader = TestLoader() + + # For GitHub Actions grouping - track if we have an open group + self.github_actions_group_open = False if is_github_actions() else None if self.args.test is not None: self.loader.load_spec(self.args.test) if self.args.tests_file is not None: @@ -769,6 +772,18 @@ def _runTest(self, test, numberOfAssertionFailed=0, prefix='', before=lambda x=N numFailed += 1 # exception should be counted as failure return numFailed + def _ensureGitHubActionsGroupOpen(self): + """Ensure a GitHub Actions group is open for passing/skipped tests""" + if self.github_actions_group_open is False: + print('::group::✅ Passing/Skipped Tests') + self.github_actions_group_open = True + + def _closeGitHubActionsGroup(self): + """Close the current GitHub Actions group if one is open""" + if self.github_actions_group_open is True: + print('::endgroup::') + self.github_actions_group_open = False + def printSkip(self, name): print('%s:\r\n\t%s' % (Colors.Cyan(name), Colors.Green('[SKIP]'))) @@ -812,12 +827,12 @@ def run_single_test(self, test, on_timeout_func): except unittest.SkipTest: self.printSkip(test.name) - return 0 + return 0, True except Exception as e: self.printException(e) self.addFailure(test.name + " [__init__]") - return 0 + return 0, False failures = 0 before = getattr(obj, 'setUp', lambda x=None: None) @@ -842,7 +857,7 @@ def run_single_test(self, test, on_timeout_func): if failures > 0 and Defaults.print_verbose_information_on_failure: verboseInfo['after_dispose'] = lastEnv.getInformationAfterDispose() lastEnv.debugPrint(json.dumps(verboseInfo, indent=2).replace('\\n', '\n'), force=True) - return done + return done, failures == 0 def print_failures(self): for group, failures in self.testsFailed.items(): @@ -913,7 +928,28 @@ def on_timeout(): # we must update the bar anyway to see output bar.__next__() - done += self.run_single_test(test, on_timeout) + # Capture output if not disabled + if self.args.no_output_catch: + # No output capturing - run test directly + count, _ = self.run_single_test(test, on_timeout) + done += count + else: + # Capture output and print with proper grouping + output = io.StringIO() + with redirect_stdout(output): + count, passed = self.run_single_test(test, on_timeout) + done += count + + # Print captured output with proper grouping + captured = output.getvalue() + if captured: + if not passed: + # Close group before printing failure output + self._closeGitHubActionsGroup() + else: + # Ensure group is open for passing test output + self._ensureGitHubActionsGroupOpen() + print(captured, end='') self.takeEnvDown(fullShutDown=True) @@ -937,16 +973,17 @@ def on_timeout(): except Exception as e: self.handleFailure(testFullName=test.name, testname=test.name, error_msg=Colors.Bred('Exception on timeout function %s' % str(e))) finally: - results.put({'test_name': test.name, "output": output.getvalue()}, block=False) + results.put({'test_name': test.name, "output": output.getvalue(), 'result': 'timeout'}, block=False) summary.put({'done': done, 'failures': self.testsFailed}, block=False) # After we return the processes will be killed, so we must make sure the queues are drained properly. results.close() summary.close() summary.join_thread() results.join_thread() - done += self.run_single_test(test, on_timeout) + count, passed = self.run_single_test(test, on_timeout) + done += count - results.put({'test_name': test.name, "output": output.getvalue()}, block=False) + results.put({'test_name': test.name, "output": output.getvalue(), 'passed': passed}, block=False) self.takeEnvDown(fullShutDown=True) @@ -980,6 +1017,11 @@ def on_timeout(): if not has_live_processor: raise Exception('Failed to get job result and no more processors is alive') output = res['output'] + passed = res['passed'] + if not passed: + self._closeGitHubActionsGroup() + else: + self._ensureGitHubActionsGroupOpen() print('%s' % output, end="") for p in processes: diff --git a/RLTest/utils.py b/RLTest/utils.py index dd6705d..5ed25e3 100644 --- a/RLTest/utils.py +++ b/RLTest/utils.py @@ -6,6 +6,12 @@ import redis import itertools + +def is_github_actions(): + """Check if running in GitHub Actions environment""" + return os.getenv('GITHUB_ACTIONS') == 'true' + + def wait_for_conn(conn, proc, retries=20, command='PING', shouldBe=True): """Wait until a given Redis connection is ready""" err1 = '' From 287d5a7ef13ab12894293625ecb202269bff9d6c Mon Sep 17 00:00:00 2001 From: GuyAv46 <47632673+GuyAv46@users.noreply.github.com> Date: Sun, 16 Nov 2025 11:56:57 +0200 Subject: [PATCH 02/11] Add message and depth parameters to all Query assertions (#237) * add depth and message for Query, and some related fixes * remove unused import --- RLTest/env.py | 49 ++++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/RLTest/env.py b/RLTest/env.py index b31a26b..925fa62 100644 --- a/RLTest/env.py +++ b/RLTest/env.py @@ -4,7 +4,6 @@ import contextlib import inspect import os -import sys import unittest import warnings @@ -28,7 +27,7 @@ def method(*argc, **nargs): class Query: - def __init__(self, env, *query, **options): + def __init__(self, env: 'Env', *query, **options): self.query = query self.options = options self.env = env @@ -67,40 +66,40 @@ def map(self, fn): self.res = list(map(fn, self.res)) return self - def equal(self, expected): - self.env.assertEqual(self.res, expected, 1) + def equal(self, expected, depth=0, message=None): + self.env.assertEqual(self.res, expected, 1 + depth, message=message) return self - def noEqual(self, expected): - self.env.assertNotEqual(self.res, expected, 1) + def noEqual(self, expected, depth=0, message=None): + self.env.assertNotEqual(self.res, expected, 1 + depth, message=message) return self - def true(self): - self.env.assertTrue(self.res, 1) + def true(self, depth=0, message=None): + self.env.assertTrue(self.res, 1 + depth, message=message) return self - def false(self): - self.env.assertFalse(self.res, 1) + def false(self, depth=0, message=None): + self.env.assertFalse(self.res, 1 + depth, message=message) return self - def ok(self): - self.env.assertEqual(self.res, 'OK', 1) + def ok(self, depth=0, message=None): + self.env.assertEqual(self.res, 'OK', 1 + depth, message=message) return self - def contains(self, val): - self.env.assertContains(val, self.res, 1) + def contains(self, val, depth=0, message=None): + self.env.assertContains(val, self.res, 1 + depth, message=message) return self - def notContains(self, val): - self.env.assertNotContains(val, self.res, 1) + def notContains(self, val, depth=0, message=None): + self.env.assertNotContains(val, self.res, 1 + depth, message=message) return self - def error(self): - self.env.assertTrue(self.errorRaised, 1) + def error(self, depth=0, message=None): + self.env.assertTrue(self.errorRaised, 1 + depth, message=message) return self - def noError(self): - self.env.assertFalse(self.errorRaised, 1) + def noError(self, depth=0, message=None): + self.env.assertFalse(self.errorRaised, 1 + depth, message=message) return self raiseError = genDeprecated('raiseError', error) @@ -382,7 +381,7 @@ def stop(self, masters = True, slaves = True): self.envRunner.stopEnv(masters, slaves) def stopEnvWithSegFault(self, masters = True, slaves = True): - self.envRunner.stopEnvWithSegFault(masters, slaves) + self.envRunner.stopEnvWithSegFault(masters, slaves) def getEnvStr(self): return self.env @@ -523,7 +522,7 @@ def exists(self, val): def assertExists(self, val, depth=0): warnings.warn("AssertExists is deprecated, use cmd instead", DeprecationWarning) - self._assertion('%s exists in db' % repr(val), self.con.exists(val), depth=0) + self._assertion('%s exists in db' % repr(val), self.con.exists(val), depth=depth) def executeCommand(self, *query, **options): warnings.warn("execute_command is deprecated, use cmd instead", DeprecationWarning) @@ -566,10 +565,10 @@ def assertResponseError(self, msg=None, contained=None): yield 1 except Exception as e: if contained: - self.assertContains(contained, str(e), depth=2) - self._assertion('Expected Response Error', True, depth=1) + self.assertContains(contained, str(e), depth=2, message=msg) + self._assertion('Expected Response Error', True, depth=1, message=msg) else: - self._assertion('Expected Response Error', False, depth=1) + self._assertion('Expected Response Error', False, depth=1, message=msg) def restartAndReload(self, shardId=None, timeout_sec=40): self.dumpAndReload(restart=True, shardId=shardId, timeout_sec=timeout_sec) From 21f46468b218b0af1418f89a732d63d224d32aee Mon Sep 17 00:00:00 2001 From: GuyAv46 <47632673+GuyAv46@users.noreply.github.com> Date: Sun, 16 Nov 2025 14:01:08 +0200 Subject: [PATCH 03/11] CI update (#238) * split nightly from generic CI * version bump actions * update poetry lock * remove usage of pkg_resources * bump poetry version * update toml and poetry lock again * fix whitespace * Rename nightly-build job to build in biweekly.yml * support version on python 3.7 * run CI on older and newer python version * add old versions back and lock poetry * fix run * another attempt * another attempt * another attempt * another attempt * another attempt * another attempt * another attempt * fix poetry lock * CI improvement * improve comment and concurrency group * add failure notification --- .github/workflows/biweekly.yml | 26 ++++++++ .github/workflows/ci.yml | 52 +++++++++------ RLTest/_version.py | 13 +++- poetry.lock | 112 ++++++++++++++++++++------------- pyproject.toml | 7 ++- 5 files changed, 144 insertions(+), 66 deletions(-) create mode 100644 .github/workflows/biweekly.yml diff --git a/.github/workflows/biweekly.yml b/.github/workflows/biweekly.yml new file mode 100644 index 0000000..56ebb6f --- /dev/null +++ b/.github/workflows/biweekly.yml @@ -0,0 +1,26 @@ +name: Biweekly + +on: + schedule: + - cron: "0 0 1,15 * *" # almost biweekly, twice a month + +jobs: + build: + uses: ./.github/workflows/ci.yml + secrets: inherit + + notify-failure: + needs: + - build + if: failure() + runs-on: ubuntu-latest + steps: + - name: Notify Failure + uses: slackapi/slack-github-action@v1 + with: + payload: | + { + "failed_workflow": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.RQE_NOTIFY_NIGHTLY_FAIL_HOOK }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5f86a4..c6729c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,13 @@ on: branches: - master pull_request: - schedule: - - cron: "0 0 * * *" + workflow_call: + +concurrency: + # if the event is a pull request, use the PR number to make PR checks cancel previous runs + # if the event is not a pull request, use the run number to make each run unique + group: CI-${{ github.event_name == 'pull_request' && github.event.number || github.run_number }} + cancel-in-progress: true jobs: build: @@ -14,24 +19,36 @@ jobs: runs-on: ${{ matrix.platform }} timeout-minutes: 40 strategy: + fail-fast: ${{ github.event_name == 'pull_request' }} matrix: - platform: ['ubuntu-22.04', 'macos-13'] - python: ['3.7', '3.8', '3.9', '3.10', '3.11'] - redis-version: ['7.0', '7.2'] - fail-fast: false + platform: ['ubuntu-latest', 'macos-latest'] + python: ['3.10', '3.14'] # min live version, latest testing + redis-version: ['7.4', '8.2'] + # ubuntu-latest no longer supports python 3.7, macos-latest no longer supports python 3.10 + include: + - platform: ubuntu-22.04 + python: '3.7' + redis-version: '7.4' + poetry-version: '1.5.1' + - platform: macos-latest + python: '3.11' + redis-version: '8.2' + exclude: + - platform: macos-latest + python: '3.10' defaults: run: shell: bash -l -eo pipefail {0} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Number of commits to fetch. 0 indicates all history for all branches and tags. with: fetch-depth: '' - name: clone redis - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Number of commits to fetch. 0 indicates all history for all branches and tags. with: fetch-depth: '' @@ -40,7 +57,7 @@ jobs: path: redis - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} architecture: x64 @@ -48,23 +65,20 @@ jobs: - name: Setup Poetry uses: snok/install-poetry@v1 with: - version: 1.5.1 + version: ${{ matrix.poetry-version || '2.2.1' }} virtualenvs-in-project: true virtualenvs-create: true installer-parallel: true - name: Cache poetry - uses: actions/cache@v3 + uses: actions/cache@v4 with: - path: ~/.cache/poetry # This path is specific to Ubuntu + path: .venv # Look to see if there is a cache hit for the corresponding requirements file key: ${{ matrix.platform }}-${{ matrix.python }}-pyproject.toml-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ matrix.platform }}-${{ matrix.python }}-pyproject.toml-${{hashFiles('pyproject.toml')}}} - - name: Install Python dependencies - run: poetry install -q + run: poetry install - name: Install Redis Server working-directory: redis @@ -189,7 +203,7 @@ jobs: --tls - name: Generate coverage report - if: matrix.python == '3.9' && matrix.platform != 'macos-13' + if: matrix.python == '3.14' && matrix.platform == 'ubuntu-latest' run: | TLS="tests/flow/tls" TLS_CERT=$TLS/redis.crt \ @@ -199,8 +213,8 @@ jobs: poetry run pytest --ignore=tests/flow --ignore=test_example.py --cov-config=.coveragerc --cov-report=xml --cov=RLTest - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - if: matrix.python == '3.9' && matrix.platform != 'macos-13' + uses: codecov/codecov-action@v4 + if: matrix.python == '3.14' && matrix.platform == 'ubuntu-latest' continue-on-error: true with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/RLTest/_version.py b/RLTest/_version.py index a8d82dd..fedc397 100644 --- a/RLTest/_version.py +++ b/RLTest/_version.py @@ -2,7 +2,14 @@ # This attribute is the only one place that the version number is written down, # so there is only one place to change it when the version number changes. try: - import pkg_resources - __version__ = pkg_resources.get_distribution('RLTest').version -except (pkg_resources.DistributionNotFound, AttributeError, ImportError): + from importlib.metadata import version +except ImportError: + try: # For Python<3.8 + from importlib_metadata import version # type: ignore + except ImportError: + version = None + +try: + __version__ = version('RLTest') +except Exception: __version__ = "99.99.99" # like redis modules diff --git a/poetry.lock b/poetry.lock index 5462ba6..26de10e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "async-timeout" @@ -6,6 +6,8 @@ version = "4.0.3" description = "Timeout context manager for asyncio programs" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "python_full_version < \"3.11.3\"" files = [ {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, @@ -20,6 +22,8 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] +markers = "sys_platform == \"win32\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -31,6 +35,7 @@ version = "7.2.7" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, @@ -98,30 +103,36 @@ files = [ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} [package.extras] -toml = ["tomli"] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "distro" -version = "1.8.0" +version = "1.9.0" description = "Distro - an OS platform information API" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ - {file = "distro-1.8.0-py3-none-any.whl", hash = "sha256:99522ca3e365cac527b44bde033f64c6945d90eb9f769703caaec52b09bbd3ff"}, - {file = "distro-1.8.0.tar.gz", hash = "sha256:02e111d1dc6a50abb8eed6bf31c3e48ed8b0830d1ea2a1b78c61765c2513fdd8"}, + {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, + {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, ] [[package]] name = "exceptiongroup" -version = "1.2.0" +version = "1.3.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.11\"" files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, + {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, + {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, ] +[package.dependencies] +typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} + [package.extras] test = ["pytest (>=6)"] @@ -131,6 +142,8 @@ version = "6.7.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.8\"" files = [ {file = "importlib_metadata-6.7.0-py3-none-any.whl", hash = "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5"}, {file = "importlib_metadata-6.7.0.tar.gz", hash = "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4"}, @@ -143,7 +156,7 @@ zipp = ">=0.5" [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +testing = ["flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\"", "pytest-perf (>=0.9.2)", "pytest-ruff"] [[package]] name = "iniconfig" @@ -151,6 +164,7 @@ version = "2.0.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -158,13 +172,14 @@ files = [ [[package]] name = "packaging" -version = "23.2" +version = "24.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, ] [[package]] @@ -173,6 +188,7 @@ version = "1.2.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, @@ -191,6 +207,7 @@ version = "4.2.0" description = "A Python Progressbar library to provide visual (yet text based) progress to long running operations." optional = false python-versions = ">=3.7.0" +groups = ["main"] files = [ {file = "progressbar2-4.2.0-py2.py3-none-any.whl", hash = "sha256:1a8e201211f99a85df55f720b3b6da7fb5c8cdef56792c4547205be2de5ea606"}, {file = "progressbar2-4.2.0.tar.gz", hash = "sha256:1393922fcb64598944ad457569fbeb4b3ac189ef50b5adb9cef3284e87e394ce"}, @@ -205,41 +222,43 @@ tests = ["flake8 (>=3.7.7)", "freezegun (>=0.3.11)", "pytest (>=4.6.9)", "pytest [[package]] name = "psutil" -version = "5.9.6" +version = "5.9.8" description = "Cross-platform lib for process and system monitoring in Python." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +groups = ["main"] files = [ - {file = "psutil-5.9.6-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d"}, - {file = "psutil-5.9.6-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c"}, - {file = "psutil-5.9.6-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28"}, - {file = "psutil-5.9.6-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017"}, - {file = "psutil-5.9.6-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c"}, - {file = "psutil-5.9.6-cp27-none-win32.whl", hash = "sha256:70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9"}, - {file = "psutil-5.9.6-cp27-none-win_amd64.whl", hash = "sha256:51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac"}, - {file = "psutil-5.9.6-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a"}, - {file = "psutil-5.9.6-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c"}, - {file = "psutil-5.9.6-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4"}, - {file = "psutil-5.9.6-cp36-cp36m-win32.whl", hash = "sha256:3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602"}, - {file = "psutil-5.9.6-cp36-cp36m-win_amd64.whl", hash = "sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa"}, - {file = "psutil-5.9.6-cp37-abi3-win32.whl", hash = "sha256:a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c"}, - {file = "psutil-5.9.6-cp37-abi3-win_amd64.whl", hash = "sha256:6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a"}, - {file = "psutil-5.9.6-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57"}, - {file = "psutil-5.9.6.tar.gz", hash = "sha256:e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a"}, + {file = "psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8"}, + {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73"}, + {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7"}, + {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36"}, + {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d"}, + {file = "psutil-5.9.8-cp27-none-win32.whl", hash = "sha256:36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e"}, + {file = "psutil-5.9.8-cp27-none-win_amd64.whl", hash = "sha256:bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631"}, + {file = "psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81"}, + {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421"}, + {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4"}, + {file = "psutil-5.9.8-cp36-cp36m-win32.whl", hash = "sha256:7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee"}, + {file = "psutil-5.9.8-cp36-cp36m-win_amd64.whl", hash = "sha256:27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2"}, + {file = "psutil-5.9.8-cp37-abi3-win32.whl", hash = "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0"}, + {file = "psutil-5.9.8-cp37-abi3-win_amd64.whl", hash = "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf"}, + {file = "psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"}, + {file = "psutil-5.9.8.tar.gz", hash = "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c"}, ] [package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +test = ["enum34 ; python_version <= \"3.4\"", "ipaddress ; python_version < \"3.0\"", "mock ; python_version < \"3.0\"", "pywin32 ; sys_platform == \"win32\"", "wmi ; sys_platform == \"win32\""] [[package]] name = "pytest" -version = "7.4.3" +version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, - {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, ] [package.dependencies] @@ -260,6 +279,7 @@ version = "4.1.0" description = "Pytest plugin for measuring coverage." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, @@ -278,6 +298,7 @@ version = "3.5.2" description = "Python Utils is a module with some convenient utilities not included with the standard Python install" optional = false python-versions = ">3.6.0" +groups = ["main"] files = [ {file = "python-utils-3.5.2.tar.gz", hash = "sha256:68198854fc276bc4b2403b261703c218e01ef564dcb072a7096ed9ea7aa5130c"}, {file = "python_utils-3.5.2-py2.py3-none-any.whl", hash = "sha256:8bfefc3430f1c48408fa0e5958eee51d39840a5a987c2181a579e99ab6fe5ca6"}, @@ -293,22 +314,23 @@ tests = ["flake8", "loguru", "pytest", "pytest-asyncio", "pytest-cov", "pytest-m [[package]] name = "redis" -version = "5.0.1" +version = "5.0.8" description = "Python client for Redis database and key-value store" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "redis-5.0.1-py3-none-any.whl", hash = "sha256:ed4802971884ae19d640775ba3b03aa2e7bd5e8fb8dfaed2decce4d0fc48391f"}, - {file = "redis-5.0.1.tar.gz", hash = "sha256:0dab495cd5753069d3bc650a0dde8a8f9edde16fc5691b689a566eda58100d0f"}, + {file = "redis-5.0.8-py3-none-any.whl", hash = "sha256:56134ee08ea909106090934adc36f65c9bcbbaecea5b21ba704ba6fb561f8eb4"}, + {file = "redis-5.0.8.tar.gz", hash = "sha256:0c5b10d387568dfe0698c6fad6615750c24170e548ca2deac10c649d463e9870"}, ] [package.dependencies] -async-timeout = {version = ">=4.0.2", markers = "python_full_version <= \"3.11.2\""} +async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} importlib-metadata = {version = ">=1.0", markers = "python_version < \"3.8\""} typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -hiredis = ["hiredis (>=1.0.0)"] +hiredis = ["hiredis (>1.0.0)"] ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] [[package]] @@ -317,6 +339,8 @@ version = "2.0.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "python_full_version <= \"3.11.0a6\"" files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, @@ -328,6 +352,8 @@ version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.11\"" files = [ {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, @@ -339,6 +365,8 @@ version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.8\"" files = [ {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, @@ -346,9 +374,9 @@ files = [ [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8 ; python_version < \"3.12\"", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\""] [metadata] -lock-version = "2.0" -python-versions = ">= 3.7.0" -content-hash = "ec234463c786c65c728a9f05c5e8ca424d70221587a9bdb7595b8a8f63b5414b" +lock-version = "2.1" +python-versions = ">=3.7.0" +content-hash = "f815756e3c5ef76cdbb29ab740e526c029b44892fd93f6b4557b48b6745aecaf" diff --git a/pyproject.toml b/pyproject.toml index d2beb7f..11c00e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,14 +19,17 @@ classifiers = [ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', 'License :: OSI Approved :: BSD License', 'Development Status :: 5 - Production/Stable' ] [tool.poetry.dependencies] -python = ">= 3.7.0" +python = ">=3.7.0" distro = "^1.5.0" -redis = "^5.0.0rc2" +redis = ">=5.0.0" psutil = "^5.9.5" pytest-cov = "^4.1.0" pytest = "^7.4" From 1baa5fa12d9fc999e132672157bbb14fa5d76c64 Mon Sep 17 00:00:00 2001 From: GuyAv46 <47632673+GuyAv46@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:08:09 +0200 Subject: [PATCH 04/11] Improve waitCluster (#239) * refactor to use cluster slots to ensure agreement * expose to Env * wait for both OK and same topo * minor improvements --- RLTest/env.py | 4 ++++ RLTest/redis_cluster.py | 44 +++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/RLTest/env.py b/RLTest/env.py index 925fa62..b488e85 100644 --- a/RLTest/env.py +++ b/RLTest/env.py @@ -397,6 +397,10 @@ def getClusterConnectionIfNeeded(self): else: return self.getConnection() + def waitCluster(self, timeout_sec=40): + if isinstance(self.envRunner, (ClusterEnv, EnterpriseRedisClusterEnv)): + self.envRunner.waitCluster(timeout_sec) + def addShardToClusterIfExists(self): if isinstance(self.envRunner, ClusterEnv): test_fname = self.testName.replace(':', '_') diff --git a/RLTest/redis_cluster.py b/RLTest/redis_cluster.py index 8c412bb..8b70313 100644 --- a/RLTest/redis_cluster.py +++ b/RLTest/redis_cluster.py @@ -48,24 +48,42 @@ def getInformationBeforeDispose(self): return [shard.getInformationBeforeDispose() for shard in self.shards] def getInformationAfterDispose(self): - return [shard.getInformationAfterDispose() for shard in self.shards] + return [shard.getInformationAfterDispose() for shard in self.shards] + + def _agreeOk(self): + ok = 0 + for shard in self.shards: + con = shard.getConnection() + try: + status = con.execute_command('CLUSTER', 'INFO') + except Exception as e: + print('got error on cluster info, will try again, %s' % str(e)) + continue + if 'cluster_state:ok' in str(status): + ok += 1 + return ok == len(self.shards) + + def _agreeSlots(self): + ok = 0 + first_view = None + for shard in self.shards: + con = shard.getConnection() + try: + slots_view = con.execute_command('CLUSTER', 'SLOTS') + except Exception as e: + print('got error on cluster slots, will try again, %s' % str(e)) + continue + if first_view is None: + first_view = slots_view + if slots_view == first_view: + ok += 1 + return ok == len(self.shards) def waitCluster(self, timeout_sec=40): st = time.time() - ok = 0 while st + timeout_sec > time.time(): - ok = 0 - for shard in self.shards: - con = shard.getConnection() - try: - status = con.execute_command('CLUSTER', 'INFO') - except Exception as e: - print('got error on cluster info, will try again, %s' % str(e)) - continue - if 'cluster_state:ok' in str(status): - ok += 1 - if ok == len(self.shards): + if self._agreeOk() and self._agreeSlots(): for shard in self.shards: try: shard.getConnection().execute_command('SEARCH.CLUSTERREFRESH') From 9eaa001916a7d99c92598fa4b518586fce06959a Mon Sep 17 00:00:00 2001 From: jonathan keinan Date: Tue, 2 Dec 2025 20:45:07 +0200 Subject: [PATCH 05/11] print rltest args --- RLTest/__main__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RLTest/__main__.py b/RLTest/__main__.py index 1578e8f..e7885a5 100644 --- a/RLTest/__main__.py +++ b/RLTest/__main__.py @@ -549,6 +549,8 @@ def __init__(self): if Defaults.env == 'enterprise-cluster' and Defaults.redis_config_file is not None: raise Exception('Redis configuration file is not supported with enterprise-cluster env') + print(f'RLTest args: {self.args}') + self.tests = [] self.testsFailed = {} self.currEnv = None From ca4194409146121b1baebb2c7a837ac3a34250ce Mon Sep 17 00:00:00 2001 From: jonathan keinan Date: Mon, 8 Dec 2025 10:11:06 +0200 Subject: [PATCH 06/11] code review comments --- RLTest/__main__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/RLTest/__main__.py b/RLTest/__main__.py index e7885a5..3bfc3a6 100644 --- a/RLTest/__main__.py +++ b/RLTest/__main__.py @@ -549,8 +549,6 @@ def __init__(self): if Defaults.env == 'enterprise-cluster' and Defaults.redis_config_file is not None: raise Exception('Redis configuration file is not supported with enterprise-cluster env') - print(f'RLTest args: {self.args}') - self.tests = [] self.testsFailed = {} self.currEnv = None @@ -816,6 +814,7 @@ def killEnvWithSegFault(self): else: self.stopEnvWithSegFault() + # return number of tests done, and if all passed def run_single_test(self, test, on_timeout_func): done = 0 with TestTimeLimit(self.args.test_timeout, on_timeout_func) as timeout_handler: @@ -975,7 +974,7 @@ def on_timeout(): except Exception as e: self.handleFailure(testFullName=test.name, testname=test.name, error_msg=Colors.Bred('Exception on timeout function %s' % str(e))) finally: - results.put({'test_name': test.name, "output": output.getvalue(), 'result': 'timeout'}, block=False) + results.put({'test_name': test.name, "output": output.getvalue(), 'passed': False}, block=False) summary.put({'done': done, 'failures': self.testsFailed}, block=False) # After we return the processes will be killed, so we must make sure the queues are drained properly. results.close() From ddfb54699ab94e1217daa8f490d0916d244a3b6f Mon Sep 17 00:00:00 2001 From: jonathan keinan Date: Mon, 8 Dec 2025 10:53:14 +0200 Subject: [PATCH 07/11] code review comments --- RLTest/__main__.py | 48 ++++++++++++++-------------------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/RLTest/__main__.py b/RLTest/__main__.py index 3bfc3a6..919d74f 100644 --- a/RLTest/__main__.py +++ b/RLTest/__main__.py @@ -772,15 +772,15 @@ def _runTest(self, test, numberOfAssertionFailed=0, prefix='', before=lambda x=N numFailed += 1 # exception should be counted as failure return numFailed - def _ensureGitHubActionsGroupOpen(self): - """Ensure a GitHub Actions group is open for passing/skipped tests""" - if self.github_actions_group_open is False: - print('::group::✅ Passing/Skipped Tests') + def _openGitHubActionsTestsGroup(self): + """Open a GitHub Actions group wrapping all tests""" + if is_github_actions() and self.github_actions_group_open is False: + print('::group::📋 Test Execution') self.github_actions_group_open = True - def _closeGitHubActionsGroup(self): - """Close the current GitHub Actions group if one is open""" - if self.github_actions_group_open is True: + def _closeGitHubActionsTestsGroup(self): + """Close the GitHub Actions tests group if one is open""" + if is_github_actions() and self.github_actions_group_open is True: print('::endgroup::') self.github_actions_group_open = False @@ -929,28 +929,7 @@ def on_timeout(): # we must update the bar anyway to see output bar.__next__() - # Capture output if not disabled - if self.args.no_output_catch: - # No output capturing - run test directly - count, _ = self.run_single_test(test, on_timeout) - done += count - else: - # Capture output and print with proper grouping - output = io.StringIO() - with redirect_stdout(output): - count, passed = self.run_single_test(test, on_timeout) - done += count - - # Print captured output with proper grouping - captured = output.getvalue() - if captured: - if not passed: - # Close group before printing failure output - self._closeGitHubActionsGroup() - else: - # Ensure group is open for passing test output - self._ensureGitHubActionsGroupOpen() - print(captured, end='') + done += self.run_single_test(test, on_timeout) self.takeEnvDown(fullShutDown=True) @@ -993,6 +972,8 @@ def on_timeout(): results = Queue() summary = Queue() + # Open group for all tests at the start (parallel execution) + self._openGitHubActionsTestsGroup() if self.parallelism == 1: run_jobs_main_thread(jobs) else : @@ -1018,11 +999,6 @@ def on_timeout(): if not has_live_processor: raise Exception('Failed to get job result and no more processors is alive') output = res['output'] - passed = res['passed'] - if not passed: - self._closeGitHubActionsGroup() - else: - self._ensureGitHubActionsGroupOpen() print('%s' % output, end="") for p in processes: @@ -1039,6 +1015,10 @@ def on_timeout(): endTime = time.time() + # Close group after all tests complete (parallel execution) + self._closeGitHubActionsTestsGroup() + + # Summary goes outside the group print(Colors.Bold('\nTest Took: %d sec' % (endTime - startTime))) print(Colors.Bold('Total Tests Run: %d, Total Tests Failed: %d, Total Tests Passed: %d' % (done, self.getFailedTestsCount(), done - self.getFailedTestsCount()))) if self.testsFailed: From 9cf73aea1a7118c50f06e393e5cfcbe4b99e80c1 Mon Sep 17 00:00:00 2001 From: jonathan keinan Date: Mon, 8 Dec 2025 10:54:28 +0200 Subject: [PATCH 08/11] fixes --- RLTest/__main__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/RLTest/__main__.py b/RLTest/__main__.py index 919d74f..11fd16c 100644 --- a/RLTest/__main__.py +++ b/RLTest/__main__.py @@ -828,12 +828,12 @@ def run_single_test(self, test, on_timeout_func): except unittest.SkipTest: self.printSkip(test.name) - return 0, True + return 0 except Exception as e: self.printException(e) self.addFailure(test.name + " [__init__]") - return 0, False + return 0 failures = 0 before = getattr(obj, 'setUp', lambda x=None: None) @@ -858,7 +858,7 @@ def run_single_test(self, test, on_timeout_func): if failures > 0 and Defaults.print_verbose_information_on_failure: verboseInfo['after_dispose'] = lastEnv.getInformationAfterDispose() lastEnv.debugPrint(json.dumps(verboseInfo, indent=2).replace('\\n', '\n'), force=True) - return done, failures == 0 + return done def print_failures(self): for group, failures in self.testsFailed.items(): @@ -953,17 +953,17 @@ def on_timeout(): except Exception as e: self.handleFailure(testFullName=test.name, testname=test.name, error_msg=Colors.Bred('Exception on timeout function %s' % str(e))) finally: - results.put({'test_name': test.name, "output": output.getvalue(), 'passed': False}, block=False) + results.put({'test_name': test.name, "output": output.getvalue()}, block=False) summary.put({'done': done, 'failures': self.testsFailed}, block=False) # After we return the processes will be killed, so we must make sure the queues are drained properly. results.close() summary.close() summary.join_thread() results.join_thread() - count, passed = self.run_single_test(test, on_timeout) + count = self.run_single_test(test, on_timeout) done += count - results.put({'test_name': test.name, "output": output.getvalue(), 'passed': passed}, block=False) + results.put({'test_name': test.name, "output": output.getvalue()}, block=False) self.takeEnvDown(fullShutDown=True) From 5a21a5b0147b387b0698e52f04c67bcd4ce02a24 Mon Sep 17 00:00:00 2001 From: jonathan keinan Date: Mon, 8 Dec 2025 10:55:06 +0200 Subject: [PATCH 09/11] small fix --- RLTest/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/RLTest/__main__.py b/RLTest/__main__.py index 11fd16c..06e7174 100644 --- a/RLTest/__main__.py +++ b/RLTest/__main__.py @@ -960,8 +960,7 @@ def on_timeout(): summary.close() summary.join_thread() results.join_thread() - count = self.run_single_test(test, on_timeout) - done += count + done += self.run_single_test(test, on_timeout) results.put({'test_name': test.name, "output": output.getvalue()}, block=False) From e39a2e326e916dcaa05869a92a2987edaacea9b4 Mon Sep 17 00:00:00 2001 From: jonathan keinan Date: Mon, 8 Dec 2025 10:56:01 +0200 Subject: [PATCH 10/11] remove redundant function call --- RLTest/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RLTest/__main__.py b/RLTest/__main__.py index 06e7174..db747f9 100644 --- a/RLTest/__main__.py +++ b/RLTest/__main__.py @@ -774,13 +774,13 @@ def _runTest(self, test, numberOfAssertionFailed=0, prefix='', before=lambda x=N def _openGitHubActionsTestsGroup(self): """Open a GitHub Actions group wrapping all tests""" - if is_github_actions() and self.github_actions_group_open is False: + if self.github_actions_group_open is False: print('::group::📋 Test Execution') self.github_actions_group_open = True def _closeGitHubActionsTestsGroup(self): """Close the GitHub Actions tests group if one is open""" - if is_github_actions() and self.github_actions_group_open is True: + if self.github_actions_group_open is True: print('::endgroup::') self.github_actions_group_open = False From 03e272096d80940019bd257ddb5b4920b2823979 Mon Sep 17 00:00:00 2001 From: kei-nan Date: Mon, 8 Dec 2025 12:18:54 +0200 Subject: [PATCH 11/11] Apply suggestions from code review Co-authored-by: GuyAv46 <47632673+GuyAv46@users.noreply.github.com> --- RLTest/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RLTest/utils.py b/RLTest/utils.py index 5ed25e3..36197d6 100644 --- a/RLTest/utils.py +++ b/RLTest/utils.py @@ -9,7 +9,7 @@ def is_github_actions(): """Check if running in GitHub Actions environment""" - return os.getenv('GITHUB_ACTIONS') == 'true' + return os.getenv('GITHUB_ACTIONS', '') != '' def wait_for_conn(conn, proc, retries=20, command='PING', shouldBe=True):