diff --git a/scripts/retriveAllTests/Readme.md b/scripts/retriveAllTests/Readme.md new file mode 100644 index 0000000..268ce2e --- /dev/null +++ b/scripts/retriveAllTests/Readme.md @@ -0,0 +1,101 @@ +# Generate Tests YAML Script + +This Python script is designed to generate a YAML structure for running tests in a CI/CD pipeline. It reads .yaml files from a specified directory (assertoor-tests), applies optional filtering, and can slice the tests into groups for parallel execution. + +## Features + +- Recursive Directory Traversal: Automatically searches for .yaml test files within the assertoor-tests directory and its subdirectories. +- Inclusion and Exclusion Filters: Allows you to specify patterns to include or exclude specific test files. +- Grouping: Splits the tests into specified groups for parallel processing. + +## Usage +To use the script, navigate to the root directory of your project and run the script using Python: + +### Basic Usage + +`python retriveAllTests.py` + +This will generate a YAML structure with all .yaml files found in the assertoor-tests directory and its subdirectories, excluding all.yaml. + +### Branch Selection +To specify a branch other than the default (master), use the --branch option: + +`python retriveAllTests.py --branch "pectra"` + +This command will use the pectra branch instead of master to retrieve the test files. + +>**IMPORTANT** +> +>This is only a branch replacement feature in path - make sure to checkout repository on proper branch before applying this parameter. + +### Filtering Tests +#### Include Specific Tests + +To include only certain tests based on their filenames, use the --include option: + +`python retriveAllTests.py --include "validator" "proposal"` + +This will include all files with validator or proposal in their filenames. + +#### Exclude Specific Tests + +To exclude certain tests based on their filenames, use the --exclude option: + +`python retriveAllTests.py --exclude "verkle" "blob"` + +This will exclude all files with verkle or blob in their filenames. + +### Slicing Tests for Parallel Execution + +To divide the list of tests into groups for parallel execution, use the --groups option: + +`python retriveAllTests.py --groups 3` + +This will split the tests into 3 groups. + +### Raw Output Usage + +To output the raw URLs without YAML formatting, use the --raw option: + +`python retriveAllTests.py --raw` + +This will output the URLs as plain text, one per line: + +``` +https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/pectra-dev/test1.yaml +https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/verkle-dev/testA.yaml +... +``` + +### Json Output Usage + +To output the test URLs in JSON format, use the --json option: + +`python retriveAllTests.py --json` + +This will output the URLs in a JSON format, like so: + +``` +{ + "1": [ + "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/pectra-dev/test1.yaml", + "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/verkle-dev/testA.yaml", + ... + ] +} +``` + +## Example Output +The script will output the assertoor_params section needed for Kurtosis network_params YAML structure in the format required for your CI/CD configuration: + +``` +assertoor_params: + tests: + - { file: "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/all-opcodes-test.yaml" } + - { file: "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/big-calldata-tx-test.yaml" } + - { file: "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/blob-transactions-test.yaml" } + - { file: "https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/block-proposal-check.yaml" } + ... +``` + +If you use the --groups option, it will output separate YAML structures for each group. diff --git a/scripts/retriveAllTests/retriveAllTests.py b/scripts/retriveAllTests/retriveAllTests.py new file mode 100644 index 0000000..5e94f52 --- /dev/null +++ b/scripts/retriveAllTests/retriveAllTests.py @@ -0,0 +1,71 @@ +import os +from pathlib import Path +import argparse +import math +import json + +script_dir = Path(__file__).resolve().parent +repo_dir = script_dir.parents[1] +tests_dir = repo_dir / "assertoor-tests" + +def get_yaml_files(tests_dir, branch="master", include=None, exclude=None): + yaml_files = [] + for root, _, files in os.walk(tests_dir): + for file in files: + if file.endswith(".yaml"): + full_path = Path(root) / file + + if file == "all.yaml": + continue + + if include and not any(inc in str(full_path) for inc in include): + continue + + if exclude and any(exc in str(full_path) for exc in exclude): + continue + + raw_url = f"https://raw.githubusercontent.com/ethpandaops/assertoor-test/{branch}/{full_path.relative_to(repo_dir)}" + yaml_files.append(raw_url) + + return yaml_files + +def construct_yaml_structure(yaml_files): + yaml_structure = "assertoor_params:\n" + yaml_structure += " tests:\n" + for url in yaml_files: + yaml_structure += f" - {{ file: \"{url}\" }}\n" + return yaml_structure + +def slice_tests(yaml_files, groups): + sliced_groups = [] + group_size = math.ceil(len(yaml_files) / groups) + for i in range(0, len(yaml_files), group_size): + sliced_groups.append(yaml_files[i:i + group_size]) + return sliced_groups + +def main(): + parser = argparse.ArgumentParser(description="Generate test URLs with optional filtering, slicing, and formatting.") + parser.add_argument("--branch", type=str, default="master", help="The branch name to use for constructing the raw URLs.") + parser.add_argument("--include", type=str, nargs='*', help="List of texts to include in file names (only include matching files).") + parser.add_argument("--exclude", type=str, nargs='*', help="List of texts to exclude from file names (exclude matching files).") + parser.add_argument("--groups", type=int, help="Number of groups to slice the tests into.", default=1) + parser.add_argument("--raw", action="store_true", help="Output raw URLs without YAML formatting.") + parser.add_argument("--json", action="store_true", help="Output URLs in JSON format.") + + args = parser.parse_args() + + yaml_files = get_yaml_files(tests_dir, branch=args.branch, include=args.include, exclude=args.exclude) + sliced_yaml_files = slice_tests(yaml_files, args.groups) + + if args.json: + json_output = {str(idx + 1): group for idx, group in enumerate(sliced_yaml_files)} + print(json.dumps(json_output, indent=2)) + else: + for idx, group in enumerate(sliced_yaml_files, start=1): + if args.raw: + print("\n".join(group) + "\n") + else: + print(construct_yaml_structure(group)) + +if __name__ == "__main__": + main()