diff --git a/Pipfile b/Pipfile index 1490547d6..8d78d0609 100644 --- a/Pipfile +++ b/Pipfile @@ -22,6 +22,7 @@ mypy = "<1.16.0" pre-commit = "*" responses = "*" types-click = "*" +types-dataclasses = {markers = "python_version < '3.7'"} types-pkg_resources = "0.1.3" types-python-dateutil = "*" types-requests = "*" diff --git a/launchable/commands/compare/subsets.py b/launchable/commands/compare/subsets.py index c6162f4e0..d43e547a1 100644 --- a/launchable/commands/compare/subsets.py +++ b/launchable/commands/compare/subsets.py @@ -1,42 +1,218 @@ -from typing import List, Tuple, Union +from dataclasses import dataclass +from http import HTTPStatus +from pathlib import Path +from typing import Any, Dict, Generic, List, Optional, Sequence, Tuple, TypeVar, Union import click from tabulate import tabulate +from launchable.testpath import unparse_test_path +from launchable.utils.launchable_client import LaunchableClient + + +@dataclass(frozen=True) +class SubsetResultBase: + order: int + name: str + + +@dataclass(frozen=True) +class SubsetResult(SubsetResultBase): + density: float + reason: str + duration_sec: float + + @classmethod + def from_inspect_api(cls, result: Dict[str, Any], order: int) -> "SubsetResult": + test_path = result.get("testPath", []) or [] + name = unparse_test_path(test_path) + density = float(result.get("density") or 0.0) + reason = result.get("reason", "") + duration_sec = float(result.get("duration") or 0.0) / 1000.0 # convert to sec from msec + return cls(order=order, name=name, density=density, reason=reason, duration_sec=duration_sec) + + +TSubsetResult = TypeVar("TSubsetResult", bound="SubsetResultBase") + + +class SubsetResultBases(Generic[TSubsetResult]): + def __init__(self, results: Sequence[TSubsetResult]): + self._results: List[TSubsetResult] = list(results) + self._index_map = {r.name: r.order for r in self._results} + + @property + def results(self) -> List[TSubsetResult]: + return self._results + + def get_order(self, name: str) -> Optional[int]: + return self._index_map.get(name) + + @staticmethod + def from_file(file_path: Path) -> "SubsetResultBases[SubsetResultBase]": + with open(file_path, "r", encoding="utf-8") as subset_file: + results = subset_file.read().splitlines() + entries = [SubsetResultBase(order=order, name=result) for order, result in enumerate(results, start=1)] + return SubsetResultBases(entries) + + +class SubsetResults(SubsetResultBases[SubsetResult]): + def __init__(self, results: Sequence[SubsetResult]): + super().__init__(results) + + @property + def results(self) -> List[SubsetResult]: + return super().results + + @classmethod + def load(cls, client: LaunchableClient, subset_id: int) -> "SubsetResults": + try: + response = client.request("get", f"subset/{subset_id}") + if response.status_code == HTTPStatus.NOT_FOUND: + raise click.ClickException( + f"Subset {subset_id} not found. Check subset ID and try again." + ) + response.raise_for_status() + except Exception as exc: + client.print_exception_and_recover(exc, "Warning: failed to load subset results") + raise click.ClickException("Failed to load subset results") from exc + + payload = response.json() + order = 1 + results: List[SubsetResult] = [] + entries = (payload.get("testPaths", []) or []) + (payload.get("rest", []) or []) + for entry in entries: + results.append(SubsetResult.from_inspect_api(entry, order)) + order += 1 + return cls(results) + @click.command() -@click.argument('file_before', type=click.Path(exists=True)) -@click.argument('file_after', type=click.Path(exists=True)) -def subsets(file_before, file_after): - """ - Compare two subset files and display changes in test order positions - """ - - # Read files and map test paths to their indices - with open(file_before, 'r') as f: - before_tests = f.read().splitlines() - before_index_map = {test: idx for idx, test in enumerate(before_tests)} - - with open(file_after, 'r') as f: - after_tests = f.read().splitlines() - after_index_map = {test: idx for idx, test in enumerate(after_tests)} +@click.argument('file_before', type=click.Path(exists=True), required=False) +@click.argument('file_after', type=click.Path(exists=True), required=False) +@click.option( + '--subset-id-before', + 'subset_id_before', + type=int, + help='Subset ID for the first subset to compare', + metavar="SUBSET_ID") +@click.option( + '--subset-id-after', + 'subset_id_after', + type=int, + help='Subset ID for the second subset to compare', + metavar="SUBSET_ID") +@click.pass_context +def subsets(context: click.core.Context, file_before, file_after, subset_id_before, subset_id_after): + """Compare subsets sourced from files or remote subset IDs.""" + + if (file_before is not None) ^ (file_after is not None): + raise click.ClickException("Provide both subset files when using file arguments.") + if (subset_id_before is not None) ^ (subset_id_after is not None): + raise click.ClickException("Provide both subset IDs when using --subset-id options.") + + from_file = file_before is not None and file_after is not None + from_subset_id = subset_id_before is not None and subset_id_after is not None + + if from_file and from_subset_id: + raise click.ClickException("Specify either both subset files or both subset IDs, not both.") + if not from_file and not from_subset_id: + raise click.ClickException("You must specify either both subset files or both subset IDs.") + + if from_subset_id: + + client = LaunchableClient(app=context.obj) + # for type check + assert subset_id_before is not None and subset_id_after is not None + _from_subset_ids(client=client, subset_id_before=subset_id_before, subset_id_after=subset_id_after) + return + + # for type check + assert file_before is not None and file_after is not None + _from_files(file_before=file_before, file_after=file_after) + + +def _from_subset_ids(client: LaunchableClient, subset_id_before: int, subset_id_after: int): + before_subset = SubsetResults.load(client, subset_id_before) + after_subset = SubsetResults.load(client, subset_id_after) + + total = 0 + promoted = 0 + demoted = 0 + affected = set() + # List of tuples representing test order changes + # (Rank, Subset Rank, Test Path, Reason, Density) + rows: List[Tuple[str, Union[int, str], str, str, Union[float, str]]] = [] + + # Calculate order difference and add each test in file_after to changes + for result in after_subset.results: + total += 1 + if result.reason.startswith("Changed file: "): + affected.add(result.reason[len("Changed file: "):]) + + test_name = result.name + after_order = result.order + before_order = before_subset.get_order(test_name) + if before_order is None: + rows.append(('NEW', after_order, test_name, result.reason, result.density)) + else: + diff = after_order - before_order + rank = "±0" + if diff > 0: + rank = "↓" + str(diff) + demoted += 1 + elif diff < 0: + rank = "↑" + str(-diff) + promoted += 1 + + rows.append((rank, after_order, test_name, result.reason, result.density)) + + # Add all deleted tests to changes + for result in before_subset.results: + test_name = result.name + before_order = result.order + if after_subset.get_order(test_name) is None: + rows.append(("DELETED", '-', test_name, "", "")) + + summary = f"""PTS subset change summary: +──────────────────────────────── +-> {total} tests analyzed | {promoted} ↑ promoted | {demoted} ↓ demoted +-> Code files affected: {', '.join(sorted(affected)) if len(affected) < 10 else str(len(affected)) + ' files'} +──────────────────────────────── +""" + + # Display results in a tabular format + headers = ["Δ Rank", "Subset Rank", "Test Name", "Reason", "Density"] + tabular_data = [ + (rank, after, test_name, reason, density) + for rank, after, test_name, reason, density in rows + ] + click.echo_via_pager(summary + "\n" + tabulate(tabular_data, headers=headers, tablefmt="simple")) + + +def _from_files(file_before: Path, file_after: Path): + before_subset = SubsetResultBases.from_file(file_before) + after_subset = SubsetResultBases.from_file(file_after) # List of tuples representing test order changes (before, after, diff, test) rows: List[Tuple[Union[int, str], Union[int, str], Union[int, str], str]] = [] # Calculate order difference and add each test in file_after to changes - for after_idx, test in enumerate(after_tests): - if test in before_index_map: - before_idx = before_index_map[test] - diff = after_idx - before_idx - rows.append((before_idx + 1, after_idx + 1, diff, test)) + for result in after_subset.results: + test_name = result.name + after_order = result.order + before_order = before_subset.get_order(test_name) + if before_order is not None: + diff = after_order - before_order + rows.append((before_order, after_order, diff, test_name)) else: - rows.append(('-', after_idx + 1, 'NEW', test)) + rows.append(('-', after_order, 'NEW', test_name)) # Add all deleted tests to changes - for before_idx, test in enumerate(before_tests): - if test not in after_index_map: - rows.append((before_idx + 1, '-', 'DELETED', test)) + for result in before_subset.results: + test_name = result.name + before_order = result.order + if after_subset.get_order(test_name) is None: + rows.append((before_order, '-', 'DELETED', test_name)) # Sort changes by the order diff rows.sort(key=lambda x: (0 if isinstance(x[2], str) else 1, x[2])) diff --git a/setup.cfg b/setup.cfg index d86cc5b35..1c339b8ac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,7 @@ packages = find: install_requires = click>=8.0,<8.1;python_version=='3.6' click>=8.1,<8.2;python_version>'3.6' + dataclasses;python_version=='3.6' requests>=2.25;python_version>='3.6' urllib3>=1.26 junitparser>=4.0.0 diff --git a/tests/commands/compare/test_subsets.py b/tests/commands/compare/test_subsets.py index df0676bbc..4b4723ad3 100644 --- a/tests/commands/compare/test_subsets.py +++ b/tests/commands/compare/test_subsets.py @@ -1,6 +1,9 @@ import os from unittest import mock +import responses + +from launchable.utils.http_client import get_base_url from tests.cli_test_case import CliTestCase @@ -153,3 +156,62 @@ def tearDown(self): os.remove("subset-before.txt") if os.path.exists("subset-after.txt"): os.remove("subset-after.txt") + + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + @responses.activate + def test_subsets_subset_ids(self): + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/subset/100", + json={ + "subsetting": { + "id": 100, + }, + "testPaths": [ + {"testPath": [{"type": "file", "name": "aaa.py"}], "duration": 10, "density": 0.9, "reason": "Changed file: aaa.py"}, # noqa: E501 + {"testPath": [{"type": "file", "name": "bbb.py"}], "duration": 10, "density": 0.8, "reason": "Changed file: bbb.py"} # noqa: E501 + ], + "rest": [ + {"testPath": [{"type": "file", "name": "ccc.py"}], "duration": 10, "density": 0.7, "reason": "Changed file: ccc.py"} # noqa: E501 + ] + }, + status=200 + ) + responses.add( + responses.GET, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/subset/101", + json={ + "subsetting": { + "id": 101, + }, + "testPaths": [ + {"testPath": [{"type": "file", "name": "ddd.py"}], "duration": 10, "density": 0.9, "reason": "Changed file: ddd.py"}, # noqa: E501 + {"testPath": [{"type": "file", "name": "ccc.py"}], "duration": 10, "density": 0.7, "reason": "Changed file: ccc.py"} # noqa: E501 + ], + "rest": [ + {"testPath": [{"type": "file", "name": "bbb.py"}], "duration": 10, "density": 0.5, "reason": "Changed file: bbb.py"} # noqa: E501 + ] + }, + status=200 + ) + + result = self.cli('compare', 'subsets', + '--subset-id-before', '100', + '--subset-id-after', '101', + mix_stderr=False) + + self.assert_success(result) + expect = """PTS subset change summary: +──────────────────────────────── +-> 3 tests analyzed | 1 ↑ promoted | 1 ↓ demoted +-> Code files affected: bbb.py, ccc.py, ddd.py +──────────────────────────────── + +Δ Rank Subset Rank Test Name Reason Density +-------- ------------- ----------- -------------------- --------- +NEW 1 file=ddd.py Changed file: ddd.py 0.9 +↑1 2 file=ccc.py Changed file: ccc.py 0.7 +↓1 3 file=bbb.py Changed file: bbb.py 0.5 +DELETED - file=aaa.py +""" + self.assertEqual(result.stdout, expect)