diff --git a/.gitignore b/.gitignore index 2f372208..fe44eda5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ /support/default_validator/default_validator /support/interactive/interactive build/ +.vscode/ \ No newline at end of file diff --git a/problemtools/ProblemPlasTeX/__init__.py b/problemtools/ProblemPlasTeX/__init__.py index f7fb3d53..eb5ef76c 100644 --- a/problemtools/ProblemPlasTeX/__init__.py +++ b/problemtools/ProblemPlasTeX/__init__.py @@ -95,6 +95,7 @@ def render(self, document): if templatepath == None: raise Exception('Could not find templates needed for conversion to HTML') + assert templatepath is not None # Ugly but unfortunately PlasTeX is quite inflexible when it comes to # configuring where to search for template files os.environ['ProblemRendererTEMPLATES'] = templatepath diff --git a/problemtools/config/2023-07_config_specification.yaml b/problemtools/config/2023-07_config_specification.yaml new file mode 100644 index 00000000..9a3b8b60 --- /dev/null +++ b/problemtools/config/2023-07_config_specification.yaml @@ -0,0 +1,213 @@ +type: object +required: [problem_format_version,name,uuid] +properties: + problem_format_version: + type: string + default: "2023-07" + alternatives: + "2023-07": {} + type: + type: object + parsing: type-2023-07 + properties: + pass-fail: + type: bool + default: True + alternatives: + True: + require: + type/scoring: False + False: {} + scoring: + type: bool + multi-pass: + type: bool + interactive: + type: bool + submit-answer: + type: bool + alternatives: + True: + require: + type/interactive: False + type/multi-pass: False + False: {} + content: + type: string + alternatives: + pass-fail: {} + scoring: {} + multi-pass: {} + interactive: {} + submit-answer: {} + name: + type: object + parsing: name-2023-07 + properties: + en: # en will always exist, if english really doesn't exist, it will be an empty string + type: string + match_properties: + "[a-z]{2,3}|[a-z]{2}-[A-Z]{2}": + type: string + uuid: + type: string + version: + type: string + credits: + type: object + parsing: credits-2023-07 + properties: + authors: + type: list + parsing: string-to-list + content: + type: string + contributors: + type: list + parsing: string-to-list + content: + type: string + testers: + type: list + parsing: string-to-list + content: + type: string + translators: + type: object + properties: {} + match_properties: + "[a-z]{2,3}|[a-z]{2}-[A-Z]{2}": + type: list + parsing: string-to-list + content: + type: string + packagers: + type: list + parsing: string-to-list + content: + type: string + acknowledgements: + type: list + content: + type: string + source: + type: list + parsing: source-2023-07 + content: + type: object + parsing: source-item-2023-07 + properties: + name: + type: string + url: + type: string + license: + type: string + default: unknown + alternatives: + unknown: + warn: License is unknown + require: + rights_owner: ".+" + cc0|cc by|cc by-sa|educational|permission: + require: + rights_owner: ".+" + public domain: + forbid: + rights_owner: ".+" + rights_owner: + type: string + parsing: rights-owner-2023-07 + embargo_until: + type: string + alternatives: + "": {} + "\\d{4}-\\d{2}-\\d{2}(T\\d{2}:\\d{2}:\\d{2}Z)?": {} + limits: + type: object + properties: + time_multipliers: + type: object + properties: + ac_to_time_limit: + type: float + default: 2.0 + alternatives: + "1.0:": {} + time_limit_to_tle: + type: float + default: 1.5 + alternatives: + "1.0:": {} + time_limit: + type: float + default: 0.0 # Hmm, time limit needs to be calculated by default + alternatives: + "0.0:": {} + time_resolution: + type: float + default: 1.0 + alternatives: + "0.0:": {} + memory: + type: int + default: copy-from:system_default/memory + alternatives: + "1:": {} + output: + type: int + default: copy-from:system_default/output + alternatives: + "1:": {} + code: + type: int + default: copy-from:system_default/code + alternatives: + "1:": {} + compilation_time: + type: int + default: copy-from:system_default/compilation_time + alternatives: + "1:": {} + compilation_memory: + type: int + default: copy-from:system_default/compilation_memory + alternatives: + "1:": {} + validation_time: + type: int + default: copy-from:system_default/validation_time + alternatives: + "1:": {} + validation_memory: + type: int + default: copy-from:system_default/validation_memory + alternatives: + "1:": {} + validation_output: + type: int + default: copy-from:system_default/validation_output + alternatives: + "1:": {} + validation_passes: + type: int + default: 2 + alternatives: + "2:": {} + keywords: + type: list + content: + type: string + languages: + type: list + parsing: languages-parsing + content: + type: string + allow_file_writing: + type: bool + constants: + type: object + properties: {} + match_properties: + "[a-zA-Z_][a-zA-Z0-9_]*": + type: string # In spec, this should be allowed to be int or float as well... This is not supported for this system diff --git a/problemtools/config/legacy_config_specification.yaml b/problemtools/config/legacy_config_specification.yaml new file mode 100644 index 00000000..25024a00 --- /dev/null +++ b/problemtools/config/legacy_config_specification.yaml @@ -0,0 +1,174 @@ +type: object +required: [] +properties: + problem_format_version: + type: string + default: legacy + alternatives: + legacy: {} + type: + type: string + default: pass-fail + alternatives: + pass-fail: + forbid: + grading/on_reject: grade + scoring: {} + name: + type: string + uuid: + type: string + author: + type: string + source: + type: string + source_url: + type: string + alternatives: + ".+": + require: + source: ".+" + "": {} + license: + type: string + default: unknown + alternatives: + unknown: + warn: License is unknown + require: + rights_owner: ".+" + cc0|cc by|cc by-sa|educational|permission: + require: + rights_owner: ".+" + public domain: + forbid: + rights_owner: ".+" + rights_owner: + type: string + parsing: rights-owner-legacy + limits: + type: object + properties: + time_multiplier: + type: float + default: 5 + alternatives: + "0.0:": {} + time_safety_margin: + type: float + default: 2 + memory: + type: int + default: copy-from:system_default/memory + alternatives: + "1:": {} + output: + type: int + default: copy-from:system_default/output + alternatives: + "1:": {} + code: + type: int + default: copy-from:system_default/code + alternatives: + "1:": {} + compilation_time: + type: int + default: copy-from:system_default/compilation_time + alternatives: + "1:": {} + compilation_memory: + type: int + default: copy-from:system_default/compilation_memory + alternatives: + "1:": {} + validation_time: + type: int + default: copy-from:system_default/validation_time + alternatives: + "1:": {} + validation_memory: + type: int + default: copy-from:system_default/validation_memory + alternatives: + "1:": {} + validation_output: + type: int + default: copy-from:system_default/validation_output + alternatives: + "1:": {} + validation: + type: object + parsing: legacy-validation + properties: + type: + type: string + default: default + alternatives: + default: + forbid: + validation/interactive: true + validation/score: true + custom: {} + interactive: + type: bool + score: + type: bool + validator_flags: + type: string + keywords: + type: list + parsing: space-separated-strings + content: + type: string + grading: + type: object + properties: + show_test_data_groups: + type: bool + alternatives: + True: + forbid: + type: pass-fail + False: {} + on_reject: + type: string + default: worst_error + flags: + - deprecated + alternatives: + first_error: {} + worst_error: {} + grade: {} + objective: + type: string + default: max + alternatives: + min: {} + max: {} + accept_score: + type: float # Should actually be type string, add custom parsing? + default: 1.0 + flags: + - deprecated + reject_score: + type: float # Should actually be type string, add custom parsing? + default: 0.0 + flags: + - deprecated + custom_scoring: + type: bool + default: copy-from:validation/score + range: + type: object + parsing: min-max-float-string + flags: + - deprecated + properties: + min: + type: float + default: -.inf + max: + type: float + default: .inf + diff --git a/problemtools/config/system_default.yaml b/problemtools/config/system_default.yaml new file mode 100644 index 00000000..2083f01e --- /dev/null +++ b/problemtools/config/system_default.yaml @@ -0,0 +1,8 @@ +memory: 2048 +output: 8 +code: 128 +compilation_time: 60 +compilation_memory: 2048 +validation_time: 60 +validation_memory: 2048 +validation_output: 8 \ No newline at end of file diff --git a/problemtools/config_parser/__init__.py b/problemtools/config_parser/__init__.py new file mode 100644 index 00000000..841b27de --- /dev/null +++ b/problemtools/config_parser/__init__.py @@ -0,0 +1,225 @@ +from __future__ import annotations +import re +from typing import Any, Callable, Generator +from collections import deque +from copy import deepcopy +from .general import SpecificationError +from .config_path import is_copyfrom +from .parser import type_mapping +from .config_path import Path, PathError +from .parser import Parser, DefaultObjectParser +from .matcher import AlternativeMatch +from itertools import chain + + +class Metadata: + def __init__(self, specification: dict) -> None: + self.spec = specification + self.error_func = lambda s: print(f"ERROR: {s}") + self.warning_func = lambda s: print(f"WARNING: {s}") + self.data = None + + def __getitem__(self, key: str | Path) -> Any: + if self.data is None: + raise Exception("data has not been loaded yet") + if type(key) is str: + return Path.parse(key).index(self.data) + elif type(key) is Path: + return key.index(self.data) + raise Exception(f"Invalid type for indexing data ({type(key)}). Type should be string or Path") + + def set_error_callback(self, fun: Callable): + self.error_func = fun + + def set_warning_callback(self, fun: Callable): + self.warning_func = fun + + @staticmethod + def invert_graph(dependency_graph: dict[Path, list[Path]]) -> dict[Path, list[Path]]: + nodes = set(dependency_graph.keys()).union(chain(*dependency_graph.values())) + depends_on_graph: dict[Path, list[Path]] = {k: [] for k in nodes} + for dependant, dependencies in dependency_graph.items(): + for dependency in dependencies: + depends_on_graph[dependency].append(dependant) + return depends_on_graph + + @staticmethod + def topo_sort(dependency_graph: dict[Path, list[Path]]) -> Generator[Path, None, None]: + # Dependancy graph: + # Path : [Depends on Path, ...] + + in_degree = {p: len(l) for p, l in dependency_graph.items()} + + q = deque(p for p in dependency_graph.keys() if in_degree[p] == 0) + + depends_on_graph = Metadata.invert_graph(dependency_graph) + + while q: + current = q.popleft() + yield current + + for dependency in depends_on_graph[current]: + in_degree[dependency] -= 1 + + if in_degree[dependency] == 0: + q.append(dependency) + + if any(x > 0 for x in in_degree.values()): + nodes = ", ".join( + str(path) for path, count in in_degree.items() if count > 0 + ) + raise SpecificationError( + f"Unresolvable, cyclic dependencies involving ({nodes})" + ) + + def get_path_dependencies(self) -> dict[Path, list[Path]]: + stack = [(Path(), Path("properties", prop)) for prop in self.spec["properties"]] + graph = {} + while stack: + parent, p = stack.pop() + spec = p.index(self.spec) + deps = [d.spec_path() for d in Parser.get_parser_type(spec).get_dependencies()] + if len(parent.path) > 0: + deps.append(parent) + graph[p] = deps + for dep in deps: + assert dep != p + if spec["type"] == "object": + stack.extend( + (p, Path.combine(p, "properties", prop)) + for prop in spec["properties"] + ) + elif spec["type"] == "list": + stack.append((p, Path.combine(p, "content"))) + return graph + + def get_copy_dependencies(self) -> dict[Path, list[Path]]: + if self.data is None: + raise SpecificationError("Data is not loaded yet") + + stack = [(Path(), Path(child)) for child in self.data.keys()] + graph: dict[Path, list[Path]] = {Path(): []} + + while stack: + parent, path = stack.pop() + val = path.index(self.data) + graph[parent].append(path) + deps = [] + if is_copyfrom(val): + deps.append(val[1]) + graph[path] = deps + if type(val) is dict: + stack.extend((path, Path.combine(path, child)) for child in val.keys()) + elif type(val) is list: + stack.extend((path, Path.combine(path, i)) for i in range(len(val))) + + return graph + + @staticmethod + def resolve_match_properties(spec: dict, data: dict): + if spec.get("type") == "list": + if type(data) is list: + for item in data: + Metadata.resolve_match_properties(spec["content"], item) + return + if spec.get("type") != "object" or type(data) is not dict: + return + + cur_props = set(spec.get("properties", {}).keys()) + regex_props = {re.compile(pattern): desc for pattern, desc in spec.get("match_properties", {}).items()} + for prop in data.keys(): + if prop in cur_props: + continue + matching = [desc for regex, desc in regex_props.items() if regex.fullmatch(prop)] + if len(matching) > 1: + raise SpecificationError(f'Multiple match_properties could match property name "{prop}"') + elif len(matching) == 1: + spec["properties"][prop] = deepcopy(matching[0]) + + for prop_name, prop in spec.get("properties", {}).items(): + if prop_name in data: + Metadata.resolve_match_properties(prop, data[prop_name]) + + @staticmethod + def remove_match_properties(spec: dict): + if spec.get("type") == "list": + Metadata.remove_match_properties(spec["content"]) + if spec.get("type") != "object": + return + if "match_properties" in spec: + del spec["match_properties"] + for prop in spec.get("properties", {}).values(): + Metadata.remove_match_properties(prop) + + def load_config(self, config: dict, injected_data: dict) -> None: + self.data = DefaultObjectParser( + config, self.spec, Path(), self.warning_func, self.error_func + ).parse() + + if self.data is None: + raise SpecificationError("DefaultObjectParser returned None") + + Metadata.resolve_match_properties(self.spec, self.data) + Metadata.remove_match_properties(self.spec) + for cfg_path in Metadata.topo_sort(self.get_path_dependencies()): + spec = cfg_path.index(self.spec) + for full_path in cfg_path.data_paths(self.data): + parser = Parser.get_parser_type(spec)( + self.data, self.spec, full_path, self.warning_func, self.error_func + ) + full_path.set(self.data, parser.parse()) + self.data.update(injected_data) + + for full_path in Metadata.topo_sort(self.get_copy_dependencies()): + val = full_path.index(self.data) + if is_copyfrom(val): + if any(type(part) is int for part in val[1].path): + raise SpecificationError( + f"copy-from directives may not copy from lists (property: {full_path}, copy-property: {val[1]})" + ) + copy_val = deepcopy(val[1].index(self.data)) + if copy_val is None: + raise SpecificationError( + f"copy-from directive returned None (property: {full_path}, copy-property: {val[1]})" + ) + if not isinstance( + copy_val, + type_mapping[full_path.spec_path().index(self.spec)["type"]], + ): + raise SpecificationError( + f"copy-from directive provided the wrong type (property: {full_path}, copy-property: {val[1]})" + ) + full_path.set(self.data, copy_val) + + def do_alternative_checks(self, checks: dict, prop_path: Path) -> None: + if self.data is None: + raise SpecificationError("Data is not loaded yet") + if "warn" in checks: + self.warning_func(checks["warn"]) + for check_result, check_name in ((False, "forbid"), (True, "require")): + for path, matchstr in checks.get(check_name, {}).items(): + path = Path.parse(path) + val = path.index(self.data) + typ = Path.combine(path.spec_path(), "type").index(self.spec) + if AlternativeMatch.get_matcher(typ, matchstr).check(val) is not check_result: + self.error_func(f'Property {prop_path} with value {prop_path.index(self.data)} {check_name}s property {path} to match value {matchstr}') + + def check_config(self) -> None: + if self.data is None: + raise Exception('Data has not been loaded yet') + def check(spec: dict, data: Any, path: Path): + if spec["type"] == "object": + for p in spec["properties"]: + check(spec["properties"][p], data[p], Path.combine(path, p)) + elif spec["type"] == "list": + for i, cont in enumerate(data): + check(spec["content"], cont, Path.combine(path, i)) + if "alternatives" in spec: + for matchstr, checks in spec["alternatives"].items(): + if AlternativeMatch.get_matcher(spec["type"], matchstr).check(data): + self.do_alternative_checks(checks, path) + check(self.spec, self.data, Path()) + + + + diff --git a/problemtools/config_parser/config_path.py b/problemtools/config_parser/config_path.py new file mode 100644 index 00000000..ec098240 --- /dev/null +++ b/problemtools/config_parser/config_path.py @@ -0,0 +1,152 @@ +from __future__ import annotations +import re +from typing import Any + +class PathError(Exception): + pass + +def is_copyfrom(val: Any) -> bool: + return isinstance(val, tuple) and val[0] == "copy-from" + +class Path: + """Class for indexing nested dictionaries, that may also contain lists. + The text version separates keys by "/". + To index a list, use list[index]. An example of a string path is "/foo/bar[3]/baz", + which means indexing as dict["foo"]["bar"][3]["baz"]. + """ + DICT_MATCH = re.compile(r"[A-Za-z_\-][A-Za-z_\d\-]*") + LIST_MATCH = re.compile(r"\d+") + + @staticmethod + def parse(path: str) -> Path: + def parse_part(part: str): + if Path.DICT_MATCH.match(part): + return part + elif Path.LIST_MATCH.match(part): + return int(part) + raise PathError(f'Could not parse path: "{path}"') + return Path(*(parse_part(p) for p in re.sub(r'\[(\d+)\]', r'/\1', path).split("/"))) + + @staticmethod + def combine(*parts: str | int | Path) -> Path: + """Fuse multiple paths together into one path""" + res: list[str | int] = [] + for part in parts: + if isinstance(part, int): + res.append(part) + elif isinstance(part, str): + res.extend(Path.parse(part).path) + elif isinstance(part, Path): + res.extend(list(part.path)) + else: + raise PathError(f'Unknown type in parts: {type(part)}') + return Path(*res) + + def __init__(self, *path: str | int) -> None: + for p in path: + if isinstance(p, int): + if p < 0: + raise PathError('Indexes should be positive') + elif isinstance(p, str): + if not Path.DICT_MATCH.match(p): + raise PathError(f'Invalid dictionary-key: "{p}"') + else: + raise PathError(f'Invalid type for path: "{type(p)}"') + self.path = path + + def index(self, data: dict, fallback: Any = ...) -> Any: + rv = data + for part in self.path: + if isinstance(part, int): + if not isinstance(rv, list): + if fallback == ...: + raise PathError(f'Tried to index non-list type with an integer ({self.path})') + return fallback + try: + rv = rv[part] + except IndexError: + if fallback == ...: + raise PathError(f'Tried to index list out of range ({self.path})') + return fallback + else: + if part not in rv: + if fallback == ...: + raise PathError(f'Tried to access invalid key "{part}" ({self.path})') + return fallback + rv = rv[part] + return rv + + def set(self, data: dict, value): + if self == Path(): + raise PathError('Can not set root of dictionary with Path') + self.up(1).index(data)[self.last_name()] = value + + def spec_path(self) -> Path: + """Get corresponding specification-path to property""" + res = [] + for part in self.path: + if isinstance(part, str): + res.append("properties") + res.append(part) + elif isinstance(part, int): + res.append("content") + return Path(*res) + + def data_paths(self, data: dict) -> list[Path]: + """Finds all data paths that a spec_path is pointing towards (meaning it will explore all items in lists)""" + + def path_is_not_copyfrom(path: Path) -> bool: + return not is_copyfrom(path.index(data, None)) + + out = [Path()] + state = "base" + for part in self.path: + if state == "base": + if part == "properties": + state = "object-properties" + elif part == "content": + state = "base" + new_out: list[Path] = [] + for path in out: + val = path.index(data) or [] + if is_copyfrom(val): # skip copied + continue + assert isinstance(val, list) + new_out.extend(Path.combine(path, i) for i in range(len(val))) + out = new_out + if len(out) == 0: + return [] + else: + assert False + elif state == "object-properties": + combined_paths = [Path.combine(path, part) for path in out] + out = [*filter(path_is_not_copyfrom, combined_paths)] + state = 'base' + + return out + + def up(self, levels=1) -> Path: + assert levels > 0 + return Path(*self.path[:-levels]) + + def last_name(self) -> int | str: + return self.path[-1] + + def __str__(self) -> str: + strings: list[str] = [] + for part in self.path: + if isinstance(part, int): + strings[-1] += f"[{part}]" + else: + strings.append(part) + return "/" + "/".join(strings) + + def __repr__(self): + return f"Path({self})" + + def __eq__(self, value): + return self.path == value.path + + def __hash__(self): + return hash(self.path) + diff --git a/problemtools/config_parser/general.py b/problemtools/config_parser/general.py new file mode 100644 index 00000000..08e66563 --- /dev/null +++ b/problemtools/config_parser/general.py @@ -0,0 +1,4 @@ +from typing import Any + +class SpecificationError(Exception): + pass diff --git a/problemtools/config_parser/matcher.py b/problemtools/config_parser/matcher.py new file mode 100644 index 00000000..df62bce7 --- /dev/null +++ b/problemtools/config_parser/matcher.py @@ -0,0 +1,128 @@ +from .general import SpecificationError +import re +from typing import Any + +class AlternativeMatch: + def __init__(self, matchstr): + raise NotImplementedError("Specialize in subclass") + + def check(self, val: Any) -> bool: + raise NotImplementedError("Specialize in subclass") + + @staticmethod + def get_matcher(type: str, matchstr: Any) -> "AlternativeMatch": + matchers = { + "string": StringMatch, + "int": IntMatch, + "float": FloatMatch, + "bool": BoolMatch, + } + if type not in matchers: + raise SpecificationError(f'There is no matcher for type "{matchstr}"') + return matchers[type](matchstr) + + +class StringMatch(AlternativeMatch): + def __init__(self, matchstr: str): + if type(matchstr) is not str: + raise SpecificationError(f'String match needs argument to be of type string. Got {type(matchstr)}') + self.regex = re.compile(matchstr) + + def check(self, val: Any) -> bool: + return type(val) is str and bool(self.regex.fullmatch(val)) + + def __str__(self) -> str: + return self.regex.pattern + + + +class IntMatch(AlternativeMatch): + def __init__(self, matchstr: str | int): + self.start: int | None = None + self.end: int | None = None + if type(matchstr) not in (str, int): + raise SpecificationError(f"Int match needs argument to be of type string or int. Got {type(matchstr)}") + if type(matchstr) is int: + self.start = self.end = matchstr + return + assert type(matchstr) is str + try: + if matchstr.count(":") > 1: + raise ValueError + if ":" in matchstr: + + self.start, self.end = [ + int(p) if p else None for p in map(str.strip, matchstr.split(":")) + ] + else: + matchstr = matchstr.strip() + if not matchstr: + raise SpecificationError("Match string for integer was left empty") + self.start = self.end = int(matchstr) + except ValueError: + raise SpecificationError( + f'Int match string should be of the form "A:B" where A and B can be parsed as ints or left empty, or a single integer, not "{matchstr}"' + ) + + def check(self, val: Any) -> bool: + if type(val) is not int: + return False + if self.start is not None: + if val < self.start: + return False + if self.end is not None: + if val > self.end: + return False + return True + + def __str__(self) -> str: + A = str(self.start) if self.start is not None else "" + B = str(self.end) if self.end is not None else "" + if A == B and A != "": + return str(A) + return f"{A}:{B}" + + +class FloatMatch(AlternativeMatch): + def __init__(self, matchstr: str): + if type(matchstr) is not str: + raise SpecificationError(f'Float match needs argument to be of type string. Got {type(matchstr)}') + try: + if matchstr.count(":") != 1: + raise ValueError + first, second = [p.strip() for p in matchstr.split(":")] + self.start = float(first) if first else float("-inf") + self.end = float(second) if second else float("inf") + except ValueError: + raise SpecificationError( + 'Float match string should be of the form "A:B" where A and B can be parsed as floats or left empty' + ) + + def check(self, val: Any) -> bool: + return type(val) is float and self.start <= val <= self.end + + def __str__(self) -> str: + A = str(self.start) if self.start != float("-inf") else "" + B = str(self.end) if self.end != float("inf") else "" + return f"{A}:{B}" + + +class BoolMatch(AlternativeMatch): + def __init__(self, matchstr: str | bool): + if not type(matchstr) in (bool, str): + raise SpecificationError(f'BoolMatch needs to recieve either type string or bool. Got {type(matchstr)}') + if type(matchstr) is bool: + self.val = matchstr + return + matchstr = matchstr.strip().lower() + if matchstr not in {"true", "false"}: + raise SpecificationError( + 'Bool match string should be either "true" or "false"' + ) + self.val = {"true": True, "false": False}[matchstr] + + def check(self, val: Any) -> bool: + return type(val) is bool and val == self.val + + def __str__(self) -> str: + return str(self.val) diff --git a/problemtools/config_parser/parser.py b/problemtools/config_parser/parser.py new file mode 100644 index 00000000..3146d0bd --- /dev/null +++ b/problemtools/config_parser/parser.py @@ -0,0 +1,590 @@ +from typing import Callable +from .config_path import Path +from .general import SpecificationError +from .matcher import AlternativeMatch +from collections import Counter + +type_mapping = { + "string": str, + "object": dict, + "list": list, + "bool": bool, + "int": int, + "float": float, +} + +type_field_mapping = { + "*": ["default", "type", "flags", "parsing"], + "string": ["alternatives"], + "bool": ["alternatives"], + "int": ["alternatives"], + "float": ["alternatives"], + "object": ["required", "properties"], + "list": ["content"], +} + + +class Parser: + NAME: str = "" + OUTPUT_TYPE: str = "" + + @staticmethod + def get_dependencies() -> list[Path]: + return [] + + def __init__( + self, + data: dict, + specification: dict, + path: Path, + warning_func: Callable, + error_func: Callable, + ): + self.data = data + self.specification = specification + self.path = path + self.spec_path = path.spec_path() + self.warning_func = warning_func + self.error_func = error_func + + if not self.NAME: + raise NotImplementedError( + "Subclasses of Parser need to set the name of the parsing rule" + ) + if not self.OUTPUT_TYPE: + raise NotImplementedError( + "Subclasses of Parser need to set the output type of the parsing rule" + ) + + required_type = self.spec_path.index(specification)["type"] + if required_type != self.OUTPUT_TYPE: + raise SpecificationError( + f"Parsing rule ({self.NAME}) for {path} outputs {self.OUTPUT_TYPE}, but the output should be of type {required_type}" + ) + + if self.OUTPUT_TYPE in ("string", "int", "float", "bool"): + alternatives = Path.combine(self.spec_path, "alternatives").index( + self.specification, + None + ) + if alternatives is None: + self.alternatives = None + else: + self.alternatives = [ + AlternativeMatch.get_matcher(self.OUTPUT_TYPE, key) for key, _ in alternatives.items() + ] + else: + self.alternatives = None + + def check_object_properties(self, val: dict): + spec: dict = self.spec_path.index(self.specification) + required_props = spec.get('required', []) + missing_props = [req for req in required_props if req not in val] + + for req in missing_props: + self.error_func(f"Missing required property: {Path.combine(self.path, req)}") + + known_props = spec.get('properties', []) + unknown_props = [prop for prop in val.keys() if prop not in known_props] + + for prop in unknown_props: + self.warning_func(f"Unknown property: {Path.combine(self.path, prop)}") + val.pop(prop, None) + + def parse(self): + out = self._parse(self.path.index(self.data, None)) + + if out is not None: + flags = Path.combine(self.spec_path, "flags").index(self.specification, []) + if "deprecated" in flags: + self.warning_func(f"deprecated property was provided ({self.path})") + + if self.OUTPUT_TYPE == "object": + self.check_object_properties(out) + + if self.alternatives is not None: + if not any(matcher.check(out) for matcher in self.alternatives): + alts = ", ".join(f'"{matcher}"' for matcher in self.alternatives) + self.error_func( + f"Property {self.path} with value {out} did not match any of the specified alternatives ({alts})" + ) + out = None + + if out is None: + fallback = Path.combine(self.spec_path, "default").index(self.specification, type_mapping[self.OUTPUT_TYPE]()) + if type(fallback) is str and fallback.startswith("copy-from:"): + fallback = ("copy-from", Path.parse(fallback.split(":")[1])) + return fallback + + if not ( + isinstance(out, tuple) or isinstance(out, type_mapping[self.OUTPUT_TYPE]) + ): + raise SpecificationError( + f'Parsing rule "{self.NAME}" did not output the correct type. Output was: {out}' + ) + return out + + def _parse(self, val): + raise NotImplementedError("Subclasses of Parse need to implement _parse()") + + @staticmethod + def smallest_edit_dist(a: str, b: list[str]) -> str: + def edit_dist(a: str, b: str) -> int: + n = len(a) + m = len(b) + dp = [[0] * (m + 1) for _ in range(n + 1)] + for i in range(n + 1): + dp[i][0] = i + for j in range(m + 1): + dp[0][j] = j + for i in range(1, n + 1): + for j in range(1, m + 1): + if a[i - 1] == b[j - 1]: + dp[i][j] = dp[i - 1][j - 1] + else: + dp[i][j] = 1 + min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]) + return dp[n][m] + best = b[0] + best_dist = edit_dist(a, best) + for s in b[1:]: + dist = edit_dist(a, s) + if dist < best_dist: + best = s + best_dist = dist + return best + + @staticmethod + def get_parser_type(specification: dict) -> type["Parser"]: + parsing_rule = specification.get("parsing") + if parsing_rule is None: + typ = specification.get("type") + + if typ is None: + had = "', '".join(specification.keys()) + raise SpecificationError( + f"Specification did not have a MUST HAVE field 'type', the provided fields were: ('{had}')" + ) + + if typ not in type_mapping: + valid = "', '".join(type_mapping.keys()) + closest = Parser.smallest_edit_dist(typ, [*type_mapping.keys()]) + raise SpecificationError( + f"Type '{typ}' is not a valid type. Did you mean: '{closest}'? Otherwise valid types are: ('{valid}')" + ) + + + fields = specification.keys() + allowed_fields = type_field_mapping.get(typ, []) + type_field_mapping["*"] + for field in fields: + if field not in allowed_fields: + raise SpecificationError( + f"Field '{field}' is not allowed for type '{typ}', did you mean '{Parser.smallest_edit_dist(field, allowed_fields)}'?" + ) + + parsing_rule = f"default-{typ}-parser" + if parsing_rule not in parsers: + raise SpecificationError(f'Parser "{parsing_rule}" is not implemented') + return parsers[parsing_rule] + + +class DefaultStringParser(Parser): + NAME = "default-string-parser" + OUTPUT_TYPE = "string" + + def _parse(self, val): + if val is None: + return None + if not isinstance(val, str): + self.warning_func( + f"Expected value of type string but got {val}, casting to string ({self.path})" + ) + val = str(val) + return val + + +class DefaultObjectParser(Parser): + NAME = "default-object-parser" + OUTPUT_TYPE = "object" + + def _parse(self, val): + if val is None: + return None + + if not isinstance(val, dict): + self.error_func(f"Expected an object, got {val} ({self.path})") + return None + + return val + + +class DefaultListParser(Parser): + NAME = "default-list-parser" + OUTPUT_TYPE = "list" + + def _parse(self, val): + if val is None: + return None + + if not isinstance(val, list): + self.error_func(f"Expected a list, got {val} ({self.path})") + return None + + return val + + +class DefaultIntParser(Parser): + NAME = "default-int-parser" + OUTPUT_TYPE = "int" + + def _parse(self, val): + if val is None: + return None + + if not isinstance(val, int): + try: + cast = int(val) + self.warning_func( + f"Expected type int, got {val}. Casting to {cast} ({self.path})" + ) + val = cast + except ValueError: + self.error_func(f"Expected a int, got {val} ({self.path})") + return None + + return val + + +class DefaultFloatParser(Parser): + NAME = "default-float-parser" + OUTPUT_TYPE = "float" + + def _parse(self, val): + if val is None: + return None + + if not isinstance(val, (int, float)): + try: + cast = float(val) + self.warning_func( + f"Expected type float, got {val}. Casting to {cast} ({self.path})" + ) + val = cast + except ValueError: + self.error_func(f"Expected a float, got {val} ({self.path})") + return None + + return float(val) + + +class DefaultBoolParser(Parser): + NAME = "default-bool-parser" + OUTPUT_TYPE = "bool" + + def _parse(self, val): + if val is None: + return None + + if isinstance(val, str): + if val.lower() in ("true", "false"): + interpretation = val.lower() == "true" + self.warning_func( + f'Expected type bool but got stringified bool: "{val}" which will be interpreted as {interpretation} ({self.path})' + ) + val = interpretation + else: + self.error_func(f'Expected type bool, but got "{val}" ({self.path})') + return None + + if not isinstance(val, bool): + self.error_func(f"Expected type bool, got {val} ({self.path})") + return None + + return val + +class RightsOwnerLegacy(Parser): + NAME = "rights-owner-legacy" + OUTPUT_TYPE = "string" + + @staticmethod + def get_dependencies() -> list[Path]: + return [Path("author"), Path("source"), Path("license")] + + def _parse(self, val): + if isinstance(val, str): + return val + + if val is None and Path("license").index(self.data) != "public domain": + author = Path("author").index(self.data) + if len(author) > 0: + return author + source = Path("source").index(self.data) + if len(source) > 0: + return source + + return None + +class LegacyValidation(Parser): + NAME = "legacy-validation" + OUTPUT_TYPE = "object" + + def _parse(self, val): + if val is None: + return None + + if not isinstance(val, str): + self.error_func(f'Property {self.path} was expected to be given as type string') + return None + + args = val.split() + if args[0] not in ("default", "custom"): + self.error_func(f'First argument of {self.path} was expected to be either "default" or "custom"') + return None + + if len(set(args)) != len(args): + self.warning_func(f'Arguments of {self.path} contains duplicate values') + + for arg in args[1:]: + if arg not in ("score", "interactive"): + self.warning_func(f'Invalid argument "{arg}" in {self.path}') + + return { + "type": args[0], + "interactive": "interactive" in args, + "score": "score" in args + } + +class SpaceSeparatedStrings(Parser): + NAME = "space-separated-strings" + OUTPUT_TYPE = "list" + + def _parse(self, val): + if val is None: + return None + + if not isinstance(val, str): + self.error_func(f'Property {self.path} was expected to be of type string') + return None + + return val.split() + +class MinMaxFloatString(Parser): + NAME = "min-max-float-string" + OUTPUT_TYPE = "object" + + def _parse(self, val): + if val is None: + return None + + if not isinstance(val, str): + self.error_func(f'Property {self.path} was expected to be of type string') + return None + + args = val.split() + if len(args) != 2: + self.error_func(f'Property {self.path} was expected to contain exactly two space-separated floats') + return None + + try: + a, b = map(float, args) + except ValueError: + self.error_func(f'Failed to parse arguments of {self.path} as floats') + + return {"min": a, "max": b} + +class Type2023_07(Parser): + NAME = "type-2023-07" + OUTPUT_TYPE = "object" + + def _parse(self, val): + if val is None: + return None + + if type(val) is str: + val = [val] + + if type(val) is not list: + self.error_func(f'Property {self.path} was expected to be of type list or a single string. Got {type(val)}') + return None + + if len(val) == 0: + self.error_func(f'Property {self.path} was empty list, but it should contain at least one element') + return None + + valid_options = {"pass-fail", "scoring", "multi-pass", "interactive", "submit-answer"} + out = {option: False for option in valid_options} + + for option in val: + if option not in valid_options: + self.error_func(f'Property {self.path} received invalid option "{option}"') + return None + else: + if out[option]: + self.error_func(f'Property {self.path} must not contain duplicate elements. Found duplicate "{option}"') + return None + out[option] = True + + return out + +class Name2023_07(Parser): + NAME = "name-2023-07" + OUTPUT_TYPE = "object" + + def _parse(self, val): + if val is None: + return None + + if type(val) is str: + return {"en": val} + + if type(val) is not dict: + self.error_func(f'Property {self.path} should be of type string or a dictionary of language-codes to strings. Got {type(val)}') + return None + + return val + +class Credits2023_07(Parser): + NAME = "credits-2023-07" + OUTPUT_TYPE = "object" + + def _parse(self, val): + if val is None: + return None + + if type(val) is str: + return {"authors": val} + + if type(val) is not dict: + self.error_func(f'Property {self.path} should be either a single string, or a dictionary') + return None + + return val + +class StringToList(Parser): + NAME = "string-to-list" + OUTPUT_TYPE = "list" + + def _parse(self, val): + if val is None: + return None + + if type(val) is str: + return [val] + + if type(val) is not list: + self.error_func(f'Property {self.path} should be either a single string or a list of strings') + return None + + return val + +class Source2023_07(Parser): + NAME = "source-2023-07" + OUTPUT_TYPE = "list" + + def _parse(self, val): + if val is None: + return None + + if type(val) is str: + return [{"name":val}] + + if type(val) is dict: + return [val] + + if type(val) is not list: + self.error_func(f'Property {self.path} should be of type string, object or a list') + return None + + return val + +class SourceItem2023_07(Parser): + NAME = "source-item-2023-07" + OUTPUT_TYPE = "object" + + def _parse(self, val): + if val is None: + return {"name": "???"} + + if type(val) is str: + return {"name": val} + + if type(val) is dict: + if "name" not in val: + self.error_func(f'Property {self.path} needs key "name"') + val["name"] = "???" + return val + + self.error_func(f'Property {self.path} should be of type string or object, got {type(val)}') + return {"name": "???"} + +class RightsOwner2023_07(Parser): + NAME = "rights-owner-2023-07" + OUTPUT_TYPE = "string" + + @staticmethod + def get_dependencies() -> list[Path]: + return [Path("credits", "authors"), Path("source", 0), Path("license")] + + def _parse(self, val): + if type(val) is str: + return val + + if val is None and Path("license").index(self.data) != "public domain": + authors = Path("credits", "authors").index(self.data) + if len(authors) > 0: + return ' and '.join(authors) + source = Path("source").index(self.data) + if len(source) > 0: + return ' and '.join(s["name"] for s in source) + + return None + +class LanguagesParsing(Parser): + NAME = "languages-parsing" + OUTPUT_TYPE = "list" + + def _parse(self, val): + if val is None: + return ("copy-from", Path("languages")) + + if type(val) is str: + if val == "all": + return ("copy-from", Path("languages")) + else: + self.error_func(f'Property {self.path} should be a list or the string "all", got "{val}"') + + if type(val) is not list: + self.error_func(f'Property {self.path} should be a list or the string "all", got {val}') + return ("copy-from", Path("languages")) + + if len(val) == 0: + self.error_func(f'Property {self.path} needs to contain at least one language') + return ("copy-from", Path("languages")) + + return val + +parser_classes = [ + DefaultObjectParser, + DefaultListParser, + DefaultStringParser, + DefaultIntParser, + DefaultFloatParser, + DefaultBoolParser, + RightsOwnerLegacy, + LegacyValidation, + SpaceSeparatedStrings, + MinMaxFloatString, + Type2023_07, + Name2023_07, + Credits2023_07, + StringToList, + Source2023_07, + SourceItem2023_07, + RightsOwner2023_07, + LanguagesParsing, +] + +parsers = { p.NAME: p for p in parser_classes } +if len(parser_classes) != len(parsers): + duplicates = '\n'.join(f' - {prop}, {cnt} occurences' for prop, cnt in Counter(c.NAME for c in parser_classes).items() if cnt > 1) + + raise NotImplementedError(f"Duplicate name(s) detected in parsers:\n{duplicates}") diff --git a/problemtools/config_parser/spec.md b/problemtools/config_parser/spec.md new file mode 100644 index 00000000..4879fd4c --- /dev/null +++ b/problemtools/config_parser/spec.md @@ -0,0 +1,125 @@ +Specification is built up of a recursive definition of items, inspired by json-schemas. + +Each item has to contain the field "type" with one of the defined types. + +# Common fields for all types +## default +The field "default" will hold the fallback-value in the case that the user-defined value +does not exist or could not be parsed. Each type will have a standard-default that will +be used if default is not specified. + +## type +Each item has to contain a type, which is one of the types defined. This decides what +type will end up there in the final result. + +## flags +List of flags that can be used to signal things about this item. +- "deprecated": if specified in user config, give deprecation-warning + +## parsing +If this item requires more complex parsing, a parsing rule can be added. This field is +a string that will map to one of the available parsing rules. A parsing rule can require +other fields to be initialized before it allows the parsing to happen. + +Parsing rules will be defined with the following information: +- name of the rule +- type of the output +- prerequisite fields that should be validated before this one + + +# Types +The following are the types available to be specified. + +## object +Standard default: {} + +Has the following properties: +- "required": List of all strictly required properties, will give an error if one is missing +- "properties": Dictionary of property-names to their types +- "match_properties": Dictionary of regex strings to their types. A step will be performed to add all properties that match here to the normal properties. + +## list +Standard default: [] + +Has the following properties: +- "content": Specification of type contained within list + +## string +Standard default: "" + +Has the following properties: +- "alternatives": See section "alternatives" + +## int +Standard default: 0 + +Properties: +- "alternatives": See section "alternatives" + +## float +Standard default: 0.0 + +Properties: +- "alternatives": See section "alternatives" + +## bool +Standard default: False + +Properties: +- "alternatives": See section "alternatives" + +# Stages +There are a couple of stages that will happen when loading and verifying config. + +1. Data is loaded and compared to format-specfication-document. Parsing rules are applied in an order that resolves dependencies. +2. External data is injected. +3. copy-from directives are executed in an order such that all are resolved. +4. Checks are performed, like the ones in the string-types alternatives. + +# alternatives +This is a property that exists on the types string, bool, ints and floats. + +The value of "alternatives" is a dictionary, with keys that indicate certain "matches", see section about "matching" for different types for more details. Each match is a key that maps to another dictionary with checks. Further info can be found about these checks in the "Checks" section. Below is an example of how the property can look on a string-parameter. + + +```yaml +type: string +default: unknown +alternatives: + unknown: + warn: License is unknown + require: + - rights_owner: ".+" + cc0|cc by|cc by-sa|educational|permission: + require: + - rights_owner: ".+" + public domain: + forbid: + - rights_owner: ".+" +``` + +## matching +The following formats are used to match the different types. +### string +For strings, matches will be given as regex strings. +### bool +For bools, the values "True" and "False" will be given and compared like expected. This check is case-insensitive. +### int +For ints, matches will be given as a string formated like "A:B", where A and B are integers. This will form an inclusive interval that is matched. A or B may be excluded to indicate that one endpoint is infinite. A single number may also be provided, which will match only that number. All numbers should be given in decimal notation. + +### float +Similar to int, but A and B are floats instead of integers. Single numbers may not be provided due to floating point imprecisions. The floats should be able to be parsed using Python's built in float parsing. + +## Checks +Each alternative may provide certain checks. The checks are described in the following subsections. Each check has a name and an argument, with the name being a key in the dictionary for that alternative, and the argument being the value for that key. + +If the value found in the parsed config does not match any value in the alternatives, this will generate an error. If alternatives is not provided in the config, this will be treated as all alternatives being okay without any checks. + +### warn +If this check is provided, a warning will be generated with the text in the argument if the alternative is matched. This can be used to give an indication that an alternative should preferrably not be used, like an unknown license. + +### require +This check ensures certain properties in the config match a certain value (see "matches" for further details about matching). The argument is a list of dictionaries which map a path to a property to the value it should match to. If it does not match, an error will be generated during the check stage. + +### forbid +Works the same as require, but instead of requiring the properties to match, it forbids them from matching. diff --git a/problemtools/tests/config_parser_tests/config/2023-07_specification.yaml b/problemtools/tests/config_parser_tests/config/2023-07_specification.yaml new file mode 100644 index 00000000..9a3b8b60 --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/2023-07_specification.yaml @@ -0,0 +1,213 @@ +type: object +required: [problem_format_version,name,uuid] +properties: + problem_format_version: + type: string + default: "2023-07" + alternatives: + "2023-07": {} + type: + type: object + parsing: type-2023-07 + properties: + pass-fail: + type: bool + default: True + alternatives: + True: + require: + type/scoring: False + False: {} + scoring: + type: bool + multi-pass: + type: bool + interactive: + type: bool + submit-answer: + type: bool + alternatives: + True: + require: + type/interactive: False + type/multi-pass: False + False: {} + content: + type: string + alternatives: + pass-fail: {} + scoring: {} + multi-pass: {} + interactive: {} + submit-answer: {} + name: + type: object + parsing: name-2023-07 + properties: + en: # en will always exist, if english really doesn't exist, it will be an empty string + type: string + match_properties: + "[a-z]{2,3}|[a-z]{2}-[A-Z]{2}": + type: string + uuid: + type: string + version: + type: string + credits: + type: object + parsing: credits-2023-07 + properties: + authors: + type: list + parsing: string-to-list + content: + type: string + contributors: + type: list + parsing: string-to-list + content: + type: string + testers: + type: list + parsing: string-to-list + content: + type: string + translators: + type: object + properties: {} + match_properties: + "[a-z]{2,3}|[a-z]{2}-[A-Z]{2}": + type: list + parsing: string-to-list + content: + type: string + packagers: + type: list + parsing: string-to-list + content: + type: string + acknowledgements: + type: list + content: + type: string + source: + type: list + parsing: source-2023-07 + content: + type: object + parsing: source-item-2023-07 + properties: + name: + type: string + url: + type: string + license: + type: string + default: unknown + alternatives: + unknown: + warn: License is unknown + require: + rights_owner: ".+" + cc0|cc by|cc by-sa|educational|permission: + require: + rights_owner: ".+" + public domain: + forbid: + rights_owner: ".+" + rights_owner: + type: string + parsing: rights-owner-2023-07 + embargo_until: + type: string + alternatives: + "": {} + "\\d{4}-\\d{2}-\\d{2}(T\\d{2}:\\d{2}:\\d{2}Z)?": {} + limits: + type: object + properties: + time_multipliers: + type: object + properties: + ac_to_time_limit: + type: float + default: 2.0 + alternatives: + "1.0:": {} + time_limit_to_tle: + type: float + default: 1.5 + alternatives: + "1.0:": {} + time_limit: + type: float + default: 0.0 # Hmm, time limit needs to be calculated by default + alternatives: + "0.0:": {} + time_resolution: + type: float + default: 1.0 + alternatives: + "0.0:": {} + memory: + type: int + default: copy-from:system_default/memory + alternatives: + "1:": {} + output: + type: int + default: copy-from:system_default/output + alternatives: + "1:": {} + code: + type: int + default: copy-from:system_default/code + alternatives: + "1:": {} + compilation_time: + type: int + default: copy-from:system_default/compilation_time + alternatives: + "1:": {} + compilation_memory: + type: int + default: copy-from:system_default/compilation_memory + alternatives: + "1:": {} + validation_time: + type: int + default: copy-from:system_default/validation_time + alternatives: + "1:": {} + validation_memory: + type: int + default: copy-from:system_default/validation_memory + alternatives: + "1:": {} + validation_output: + type: int + default: copy-from:system_default/validation_output + alternatives: + "1:": {} + validation_passes: + type: int + default: 2 + alternatives: + "2:": {} + keywords: + type: list + content: + type: string + languages: + type: list + parsing: languages-parsing + content: + type: string + allow_file_writing: + type: bool + constants: + type: object + properties: {} + match_properties: + "[a-zA-Z_][a-zA-Z0-9_]*": + type: string # In spec, this should be allowed to be int or float as well... This is not supported for this system diff --git a/problemtools/tests/config_parser_tests/config/basic_config.yaml b/problemtools/tests/config_parser_tests/config/basic_config.yaml new file mode 100644 index 00000000..bf264828 --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/basic_config.yaml @@ -0,0 +1,38 @@ +type: object +require: + - bar + - boz +properties: + foo: + flags: + - deprecated + type: int + default: 1337 + bar: + type: string + alternatives: + x: {} + y: + warn: watch out you are using y + z: + require: + baz: true + baz: + type: bool + default: True + alternatives: + True: + warn: "true is now deprecated" + False: {} + boz: + type: float + default: 3.5 + alternatives: + "3.1414:3.1416": + require: + foo: 1337 + warn: this is pie not real pi 7/2 + ":": {} # Allows all values + copied: + type: string + default: copy-from:external/cool-string diff --git a/problemtools/tests/config_parser_tests/config/complex_copies.yaml b/problemtools/tests/config_parser_tests/config/complex_copies.yaml new file mode 100644 index 00000000..fcd96f2f --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/complex_copies.yaml @@ -0,0 +1,43 @@ +type: object +properties: + a: + type: object + default: "copy-from:h/j" + properties: + k: + type: int + default: 1 + l: + type: int + default: 1 + b: + type: object + properties: + c: + type: int + default: 123 + d: + type: object + properties: + k: + type: int + default: 2 + l: + type: int + default: "copy-from:h/i" + h: + type: object + properties: + i: + type: int + default: 1000 + j: + type: object + default: "copy-from:b/d" + properties: + k: + type: int + default: 13 + l: + type: int + default: "copy-from:b/c" diff --git a/problemtools/tests/config_parser_tests/config/config_alternatives_bool_misspelled_1.yaml b/problemtools/tests/config_parser_tests/config/config_alternatives_bool_misspelled_1.yaml new file mode 100644 index 00000000..d391b0f6 --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/config_alternatives_bool_misspelled_1.yaml @@ -0,0 +1,21 @@ +type: object +require: + - bar + - boz +properties: + foo: + flags: + - deprecated + type: int + default: 69 + bar: + type: string + alternatives: + x: {} + baz: + type: bool + default: True + alternatives: + Sann: + warn: "true is now deprecated" + False: {} \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/config_alternatives_bool_misspelled_2.yaml b/problemtools/tests/config_parser_tests/config/config_alternatives_bool_misspelled_2.yaml new file mode 100644 index 00000000..3d2be4d8 --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/config_alternatives_bool_misspelled_2.yaml @@ -0,0 +1,21 @@ +type: object +require: + - bar + - boz +properties: + foo: + flags: + - deprecated + type: int + default: 69 + bar: + type: string + alternatives: + x: {} + baz: + type: bool + default: True + alternatives: + True: + warn: "true is now deprecated" + Falsk: {} \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/config_field_misspelled.yaml b/problemtools/tests/config_parser_tests/config/config_field_misspelled.yaml new file mode 100644 index 00000000..246bbfc7 --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/config_field_misspelled.yaml @@ -0,0 +1,21 @@ +type: object +require: + - bar + - boz +properties: + foo: + flags: + - deprecated + type: int + classic: 69 + bar: + type: string + alternativ: + x: {} + baz: + type: bool + defalt: True + alteratives: + True: + warn: "true is now deprecated" + False: {} \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/config_type_misspelled.yaml b/problemtools/tests/config_parser_tests/config/config_type_misspelled.yaml new file mode 100644 index 00000000..e3ac337c --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/config_type_misspelled.yaml @@ -0,0 +1,21 @@ +type: object +require: + - bar + - boz +properties: + foo: + flags: + - deprecated + typ: int + classic: 69 + bar: + typ: string + alternative: + x: {} + baz: + typer: bool + default: True + alteratives: + True: + warn: "true is now deprecated" + False: {} \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/config_type_value_misspelled.yaml b/problemtools/tests/config_parser_tests/config/config_type_value_misspelled.yaml new file mode 100644 index 00000000..920e6bdb --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/config_type_value_misspelled.yaml @@ -0,0 +1,21 @@ +type: object +require: + - bar + - boz +properties: + foo: + flags: + - deprecated + type: Integer + classic: 69 + bar: + type: Sträng + alternatives: + x: {} + baz: + type: boolean + default: True + alteratives: + True: + warn: "true is now deprecated" + False: {} \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/empty_config.yaml b/problemtools/tests/config_parser_tests/config/empty_config.yaml new file mode 100644 index 00000000..e69de29b diff --git a/problemtools/tests/config_parser_tests/config/follows_basic_config.yaml b/problemtools/tests/config_parser_tests/config/follows_basic_config.yaml new file mode 100644 index 00000000..7bcfd34f --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/follows_basic_config.yaml @@ -0,0 +1,3 @@ +bar: z +baz: True +boz: 3.5 \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/follows_config_misspelled.yaml b/problemtools/tests/config_parser_tests/config/follows_config_misspelled.yaml new file mode 100644 index 00000000..5584ca76 --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/follows_config_misspelled.yaml @@ -0,0 +1,2 @@ +bar: x +baz: True \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/legacy_config_fails_alternative_match.yaml b/problemtools/tests/config_parser_tests/config/legacy_config_fails_alternative_match.yaml new file mode 100644 index 00000000..fd904729 --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/legacy_config_fails_alternative_match.yaml @@ -0,0 +1,2 @@ +license: Mr Moneys Private License +rights_owner: Mr Money \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/legacy_config_fails_forbid.yaml b/problemtools/tests/config_parser_tests/config/legacy_config_fails_forbid.yaml new file mode 100644 index 00000000..e4bf8c5c --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/legacy_config_fails_forbid.yaml @@ -0,0 +1,2 @@ +license: public domain +rights_owner: Mr Money \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/legacy_config_fails_require.yaml b/problemtools/tests/config_parser_tests/config/legacy_config_fails_require.yaml new file mode 100644 index 00000000..e64218e6 --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/legacy_config_fails_require.yaml @@ -0,0 +1,2 @@ +license: cc by-sa +rights_owner: "" \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/legacy_specification.yaml b/problemtools/tests/config_parser_tests/config/legacy_specification.yaml new file mode 100644 index 00000000..9a85cb6e --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/legacy_specification.yaml @@ -0,0 +1,170 @@ +type: object +required: [] +properties: + problem_format_version: + type: string + default: legacy + alternatives: + legacy: {} + type: + type: string + default: pass-fail + alternatives: + pass-fail: + forbid: + grading/on_reject: grade + scoring: {} + name: + type: string + uuid: + type: string + author: + type: string + source: + type: string + source_url: + type: string + alternatives: + ".+": + require: + source: ".+" + "": {} + license: + type: string + default: unknown + alternatives: + unknown: + warn: License is unknown + require: + rights_owner: ".+" + cc0|cc by|cc by-sa|educational|permission: + require: + rights_owner: ".+" + public domain: + forbid: + rights_owner: ".+" + rights_owner: + type: string + parsing: rights-owner-legacy + limits: + type: object + properties: + time_multiplier: + type: float + default: 5 + alternatives: + "0.0:": {} + time_safety_margin: + type: float + default: 2 + memory: + type: int + default: copy-from:system_default/memory + alternatives: + "1:": {} + output: + type: int + default: copy-from:system_default/output + alternatives: + "1:": {} + code: + type: int + default: copy-from:system_default/code + alternatives: + "1:": {} + compilation_time: + type: int + default: copy-from:system_default/compilation_time + alternatives: + "1:": {} + compilation_memory: + type: int + default: copy-from:system_default/compilation_memory + alternatives: + "1:": {} + validation_time: + type: int + default: copy-from:system_default/validation_time + alternatives: + "1:": {} + validation_memory: + type: int + default: copy-from:system_default/validation_memory + alternatives: + "1:": {} + validation_output: + type: int + default: copy-from:system_default/validation_output + alternatives: + "1:": {} + validation: + type: object + parsing: legacy-validation + properties: + type: + type: string + alternatives: + default: + forbid: + validation/interactive: true + validation/score: true + custom: {} + interactive: + type: bool + score: + type: bool + validator_flags: + type: string + keywords: + type: list + parsing: space-separated-strings + content: + type: string + grading: + type: object + properties: + show_test_data_groups: + type: bool + alternatives: + True: + forbid: + type: pass-fail + False: {} + on_reject: + type: string + default: worst_error + flags: + - deprecated + alternatives: + first_error: {} + worst_error: {} + grade: {} + objective: + type: string + default: max + alternatives: + min: {} + max: {} + accept_score: + type: float # Should actually be type string, add custom parsing? + default: 1.0 + flags: + - deprecated + reject_score: + type: float # Should actually be type string, add custom parsing? + default: 0.0 + flags: + - deprecated + range: + type: object + parsing: min-max-float-string + flags: + - deprecated + properties: + min: + type: float + default: .inf + max: + type: float + default: -.inf + diff --git a/problemtools/tests/config_parser_tests/config/minimal_2023-07.yaml b/problemtools/tests/config_parser_tests/config/minimal_2023-07.yaml new file mode 100644 index 00000000..3f7d41dd --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/minimal_2023-07.yaml @@ -0,0 +1,4 @@ +problem_format_version: 2023-07 +name: Problem Name +uuid: cool-id +license: public domain \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/config/system_defaults.yaml b/problemtools/tests/config_parser_tests/config/system_defaults.yaml new file mode 100644 index 00000000..2083f01e --- /dev/null +++ b/problemtools/tests/config_parser_tests/config/system_defaults.yaml @@ -0,0 +1,8 @@ +memory: 2048 +output: 8 +code: 128 +compilation_time: 60 +compilation_memory: 2048 +validation_time: 60 +validation_memory: 2048 +validation_output: 8 \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/helper.py b/problemtools/tests/config_parser_tests/helper.py new file mode 100644 index 00000000..c02f70a2 --- /dev/null +++ b/problemtools/tests/config_parser_tests/helper.py @@ -0,0 +1,30 @@ +from problemtools.config_parser import Metadata +from yaml import safe_load +import os + +base_dir = os.path.join(os.path.dirname(__file__), 'config') + +def load_yaml(filename) -> dict: + with open(os.path.join(base_dir, filename), 'r') as f: + content = safe_load(f) + return content + +warnings = [] +errors = [] + +def warnings_add(text: str): + warnings.append(text) + +def errors_add(text: str): + errors.append(text) + +def construct_metadata(spec_file, config_file, injected_data) -> Metadata: + errors.clear() + warnings.clear() + spec = load_yaml(spec_file) + config = load_yaml(config_file) + md = Metadata(spec) + md.set_error_callback(errors_add) + md.set_warning_callback(warnings_add) + md.load_config(config, injected_data) + return md \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/test_complex_copy_from.py b/problemtools/tests/config_parser_tests/test_complex_copy_from.py new file mode 100644 index 00000000..0037d68b --- /dev/null +++ b/problemtools/tests/config_parser_tests/test_complex_copy_from.py @@ -0,0 +1,33 @@ +from helper import * +import yaml + +config = construct_metadata('complex_copies.yaml', 'empty_config.yaml', {}) + +expected = { + "a": { + "k": 2, + "l": 1000 + }, + "b": { + "c": 123, + "d": { + "k": 2, + "l": 1000 + } + }, + "h": { + "i": 1000, + "j": { + "k": 2, + "l": 1000 + } + } +} + +if config.data != expected: + print("Data did not match expected result:") + print("expected:") + print(yaml.dump(expected)) + print("got:") + print(yaml.dump(config.data)) + assert False diff --git a/problemtools/tests/config_parser_tests/test_config_alternative_directives.py b/problemtools/tests/config_parser_tests/test_config_alternative_directives.py new file mode 100644 index 00000000..87dce758 --- /dev/null +++ b/problemtools/tests/config_parser_tests/test_config_alternative_directives.py @@ -0,0 +1,23 @@ +from helper import * + +legacy_injected_data = { + "system_default": load_yaml("system_defaults.yaml") +} + +def test_require(): + md = construct_metadata('legacy_specification.yaml', 'legacy_config_fails_require.yaml', legacy_injected_data) + md.check_config() + print(errors) + assert len(errors) == 1 + +def test_forbid(): + md = construct_metadata('legacy_specification.yaml', 'legacy_config_fails_forbid.yaml', legacy_injected_data) + md.check_config() + print(errors) + assert len(errors) == 1 + +def test_alternatives(): + md = construct_metadata('legacy_specification.yaml', 'legacy_config_fails_alternative_match.yaml', legacy_injected_data) + md.check_config() + print(errors) + assert len(errors) == 1 \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/test_config_path.py b/problemtools/tests/config_parser_tests/test_config_path.py new file mode 100644 index 00000000..02eebd50 --- /dev/null +++ b/problemtools/tests/config_parser_tests/test_config_path.py @@ -0,0 +1,56 @@ +from problemtools.config_parser import Path, PathError + +def test_parse(): + assert Path.parse("a/b/c") == Path("a", "b", "c") + assert Path.parse("array[0]/key") == Path("array", 0, "key") + assert Path.parse("nested/list[2]/item") == Path("nested", "list", 2, "item") + assert Path.parse("multiple_lists[0][0][0][0]") == Path("multiple_lists", 0, 0, 0, 0) + +def test_combine(): + assert Path.combine("a", "b", "c") == Path("a", "b", "c") + assert Path.combine("list[1]", "key") == Path("list", 1, "key") + assert Path.combine(Path("x", "y"), "z") == Path("x", "y", "z") + +def test_index(): + data = {"a": {"b": [1, 2, 3]}} + assert Path("a", "b", 1).index(data) == 2 + try: + Path("a", "c").index(data) + assert False and "Should crash on invalid indexing" + except PathError: + pass + try: + Path("a", "b", 10).index(data) is None + assert False and "Should crash on invalid indexing" + except PathError: + pass + +def test_spec_path(): + assert Path("a", "b", 2).spec_path() == Path("properties", "a", "properties", "b", "content") + assert Path("x", 3, "y").spec_path() == Path("properties", "x", "content", "properties", "y") + +def test_data_paths(): + data = {"list": ["a", "b", "c"]} + path = Path("properties", "list", "content") + assert path.data_paths(data) == [Path("list", 0), Path("list", 1), Path("list", 2)] + +def test_up(): + assert Path("a", "b", "c").up() == Path("a", "b") + assert Path("x", "y", "z").up(2) == Path("x") + +def test_last_name(): + assert Path("a", "b", "c").last_name() == "c" + assert Path("list", 3).last_name() == 3 + +def test_str_repr(): + assert str(Path("a", "b", 2)) == "/a/b[2]" + assert repr(Path("x", "y")) == "Path(/x/y)" + +def test_equality_hash(): + p1 = Path("a", "b", 1) + p2 = Path("a", "b", 1) + p3 = Path("a", "b", 2) + assert p1 == p2 + assert p1 != p3 + assert hash(p1) == hash(p2) + assert hash(p1) != hash(p3) diff --git a/problemtools/tests/config_parser_tests/test_invalid_config.py b/problemtools/tests/config_parser_tests/test_invalid_config.py new file mode 100644 index 00000000..116db529 --- /dev/null +++ b/problemtools/tests/config_parser_tests/test_invalid_config.py @@ -0,0 +1,58 @@ +from problemtools.config_parser import Metadata +from problemtools.config_parser import SpecificationError +from helper import construct_metadata, warnings, errors + +def run_test(config_yaml, follows_config_yaml, starts_with_error=None, additional_assert_function=None): + """ + Run a basic test and check for raised SpecificationError message. + """ + try: + construct_metadata(config_yaml, follows_config_yaml, {}) + assert False, 'Should have raised SpecificationError' + + except SpecificationError as e: + e = str(e) + if starts_with_error: + assert e.startswith(starts_with_error) + if additional_assert_function: + assert additional_assert_function(e) + + print(f'warnings: {warnings}') + print(f'errors: {errors}') + +def test_misspelled_type(): + run_test( + 'config_type_misspelled.yaml', + 'follows_config_misspelled.yaml', + starts_with_error="Specification did not have a MUST HAVE field 'type'," + ) + +def test_misspelled_typevalue(): + run_test( + 'config_type_value_misspelled.yaml', + 'follows_config_misspelled.yaml', + starts_with_error='Type ', + additional_assert_function=lambda e: 'is not a valid type. Did you mean:' in e + ) + +def test_misspelled_field(): + run_test( + 'config_field_misspelled.yaml', + 'follows_config_misspelled.yaml', + starts_with_error='Field ', + additional_assert_function=lambda e: ' is not allowed for type ' in e + ) + +def test_misspelled_true_alternatives_1(): + run_test( + 'config_alternatives_bool_misspelled_1.yaml', + 'follows_config_misspelled.yaml', + starts_with_error='Bool match string should be either "true" or "false"' + ) + +def test_misspelled_true_alternatives_2(): + run_test( + 'config_alternatives_bool_misspelled_2.yaml', + 'follows_config_misspelled.yaml', + starts_with_error='Bool match string should be either "true" or "false"' + ) \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/test_matcher.py b/problemtools/tests/config_parser_tests/test_matcher.py new file mode 100644 index 00000000..0c601c3f --- /dev/null +++ b/problemtools/tests/config_parser_tests/test_matcher.py @@ -0,0 +1,114 @@ +from problemtools.config_parser.matcher import AlternativeMatch, BoolMatch, StringMatch, IntMatch, FloatMatch +from problemtools.config_parser.general import SpecificationError +from itertools import chain + +import pytest + +def test_bool_match(): + for matcher in (BoolMatch(True), BoolMatch("true"), BoolMatch("tRuE")): + assert matcher.check(True) is True + assert matcher.check(False) is False + assert str(matcher) == "True" + for matcher in (BoolMatch(False), BoolMatch("false"), BoolMatch("FaLsE")): + assert matcher.check(True) is False + assert matcher.check(False) is True + assert str(matcher) == "False" + for junk in ("asdsad", 123, "foobar", 0.0): + with pytest.raises(SpecificationError): + BoolMatch(junk) + +def test_string_match(): + matcher = StringMatch("f[ou][ou]") + for s in ("foo", "fuu", "fou"): + assert matcher.check(s) is True + for s in ("bar", "fooo", "fo", "bfoo"): + assert matcher.check(s) is False + for s in ("xyz", "[aA][bB]", r"#([a-fA-F0-9]{3}|[a-fA-F0-9]{6})\b"): + assert str(StringMatch(s)) == s + for junk in (1.2, 123, True): + assert matcher.check(junk) is False + with pytest.raises(SpecificationError): + StringMatch(junk) + +def test_int_match(): + matcher = IntMatch("0:") + for i in range(-10, 0): + assert matcher.check(i) is False + for i in range(0, 10): + assert matcher.check(i) is True + assert str(matcher) == "0:" + + matcher = IntMatch(":13") + for i in range(3, 14): + assert matcher.check(i) is True + for i in range(14, 24): + assert matcher.check(i) is False + assert str(matcher) == ":13" + + matcher = IntMatch("10:20") + for i in range(10, 21): + assert matcher.check(i) is True + for i in chain(range(0, 10), range(21, 30)): + assert matcher.check(i) is False + assert str(matcher) == "10:20" + + matcher = IntMatch(":") + for i in (-100, 1000000, 23, 101010, 0): + assert matcher.check(i) is True + for i in ("foo", 0.0, True, 3.5): + assert matcher.check(i) is False + assert str(matcher) == ":" + + for v in (13, "13"): + matcher = IntMatch(v) + for i in chain(range(5, 13), range(14, 20)): + assert matcher.check(i) is False + assert matcher.check(13) is True + assert str(matcher) == "13" + + for junk in (1.2, "foo", True, "13:13:13"): + assert matcher.check(junk) is False + with pytest.raises(SpecificationError): + IntMatch(junk) + + +def test_float_match(): + for v in ("0.0:", "0:"): + matcher = FloatMatch(v) + for i in range(-10, 0): + assert matcher.check(i * 0.5) is False + for i in range(1, 10): + assert matcher.check(i * 0.5) is True + + for v in (":13", ":13.0", ":1.3e1"): + matcher = FloatMatch(":13") + for i in range(10, 25): + assert matcher.check(i * 0.5) is True + for i in range(27, 30): + assert matcher.check(i * 0.5) is False + + matcher = FloatMatch("1.0:2.0") + for i in range(11, 20): + assert matcher.check(i * 0.1) is True + for i in chain(range(0, 10), range(21, 30)): + assert matcher.check(i * 0.1) is False + + matcher = FloatMatch(":") + for i in (-100, 1000000, 23, 101010, 0): + assert matcher.check(i * 1.0) is True + for i in ("foo", 0, True, 35): + assert matcher.check(i) is False + + for junk in (1.2, "foo", True, 13, "13", "13:13:13"): + print(junk) + with pytest.raises(SpecificationError): + FloatMatch(junk) + +def test_match_factory(): + for junk in (1.2, "foo", True, 13, "13", "13:13:13"): + with pytest.raises(SpecificationError): + AlternativeMatch.get_matcher(junk, "123") + assert type(AlternativeMatch.get_matcher("string", "abc123")) is StringMatch + assert type(AlternativeMatch.get_matcher("float", "0.0:1.0")) is FloatMatch + assert type(AlternativeMatch.get_matcher("int", "13:25")) is IntMatch + assert type(AlternativeMatch.get_matcher("bool", "True")) is BoolMatch \ No newline at end of file diff --git a/problemtools/tests/config_parser_tests/test_valid_config.py b/problemtools/tests/config_parser_tests/test_valid_config.py new file mode 100644 index 00000000..6a59e32a --- /dev/null +++ b/problemtools/tests/config_parser_tests/test_valid_config.py @@ -0,0 +1,43 @@ +from problemtools.config_parser import Metadata +from helper import * + +def test_basic_config(): + injected = { + 'external': { + "cool-string": "yo this string is ballin" + } + } + data = construct_metadata('basic_config.yaml', 'follows_basic_config.yaml', injected) + data.check_config() + + print(f"warnings: {warnings}") + print(f"errors: {errors}") + + assert data["foo"] == 1337 + assert data["bar"] == "z" + assert data["baz"] == True + assert abs(data["boz"] - 3.5) < 0.01 + assert data["copied"] == "yo this string is ballin" + assert len(warnings) > 0 + +legacy_injected_data = { + "system_default": load_yaml("system_defaults.yaml") +} + +def test_legacy_config_empty(): + data = construct_metadata('legacy_specification.yaml', 'empty_config.yaml', legacy_injected_data) + print(f"warnings: {warnings}") + print(f"errors: {errors}") + +injected_data_2023_07 = { + "system_default": load_yaml("system_defaults.yaml"), + "languages": ["python", "rust", "uhhh", "c++"] +} + +def test_2023_07_config_minimal(): + data = construct_metadata('2023-07_specification.yaml', 'minimal_2023-07.yaml', injected_data_2023_07) + data.check_config() + + print(f"warnings: {warnings}") + print(f"errors: {errors}") + assert len(errors) == 0 \ No newline at end of file diff --git a/problemtools/tests/test_verify_hello.py b/problemtools/tests/test_verify_hello.py index 49ff53aa..acdc638c 100644 --- a/problemtools/tests/test_verify_hello.py +++ b/problemtools/tests/test_verify_hello.py @@ -1,7 +1,6 @@ import pathlib import problemtools.verifyproblem as verify - def test_load_hello(): directory = pathlib.Path(__file__).parent / "hello" string = str(directory.resolve()) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 646e262f..6393d51b 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -32,10 +32,13 @@ from . import config from . import languages from . import run +from . import config_parser from abc import ABC from typing import Any, Callable, ClassVar, Literal, Pattern, Match, ParamSpec, Type, TypeVar +from copy import deepcopy + log = logging.getLogger(__name__) Verdict = Literal['AC', 'TLE', 'OLE', 'MLE', 'RTE', 'WA', 'PAC', 'JE'] @@ -421,9 +424,9 @@ def __init__(self, problem: Problem, aspect_name: str, datadir: str|None=None, p # TODO: Decide if these should stay # Some deprecated properties are inherited from problem config during a transition period - problem_grading = problem.get(ProblemConfig)['grading'] + problem_grading = problem.get(ProblemConfig)['original_data']['grading'] for key in ['accept_score', 'reject_score', 'range']: - if key in problem.get(ProblemConfig)['grading']: + if key in problem_grading: self.config[key] = problem_grading[key] problem_on_reject = problem_grading.get('on_reject') @@ -496,7 +499,7 @@ def get_score_range(self) -> tuple[float, float]: score_range = self.config['range'] min_score, max_score = list(map(float, score_range.split())) return (min_score, max_score) - except: + except Exception: return (float('-inf'), float('inf')) @@ -783,158 +786,70 @@ class ProblemStatement2023_07(ProblemStatement): class ProblemConfig(ProblemPart): PART_NAME = 'config' - - @staticmethod - def setup_dependencies(): - return {ProblemStatement} - - _MANDATORY_CONFIG = ['name'] - _OPTIONAL_CONFIG = config.load_config('problem.yaml') - _VALID_LICENSES = ['unknown', 'public domain', 'cc0', 'cc by', 'cc by-sa', 'educational', 'permission'] - - def setup(self): + SPECIFICATION_FILE_NAME: str|None = None + + def get_injected_data(self) -> dict: + orig_data = deepcopy(self.data) # Ugly hack to make TestCaseGroup work like before + if 'grading' not in orig_data: + orig_data['grading'] = {'objective': 'max', 'show_test_data_groups': False} + return { + "languages": self.problem.language_config, + "system_default": config.load_config('system_default.yaml'), + "original_data": orig_data + } + + def setup(self) -> dict: + if self.SPECIFICATION_FILE_NAME is None: + raise NotImplementedError("Subclasses of ProblemConfig need to define SPECIFICATION_FILE_NAME") self.debug(' Loading problem config') + spec = config.load_config(self.SPECIFICATION_FILE_NAME) self.configfile = os.path.join(self.problem.probdir, 'problem.yaml') - self._data = {} - + self.data: dict[str, Any] = {} if os.path.isfile(self.configfile): try: with open(self.configfile) as f: - self._data = yaml.safe_load(f) - # Loading empty yaml yields None, for no apparent reason... - if self._data is None: - self._data = {} + self.data = yaml.safe_load(f) or {} except Exception as e: self.error(str(e)) + self.metadata = config_parser.Metadata(spec) + self.metadata.set_error_callback(self.error) + self.metadata.set_warning_callback(self.warning) + self.metadata.load_config(self.data, self.get_injected_data()) - # Add config items from problem statement e.g. name - self._data.update(self.problem.get(ProblemStatement)) - - # Populate rights_owner unless license is public domain - if 'rights_owner' not in self._data and self._data.get('license') != 'public domain': - if 'author' in self._data: - self._data['rights_owner'] = self._data['author'] - elif 'source' in self._data: - self._data['rights_owner'] = self._data['source'] - - if 'license' in self._data: - self._data['license'] = self._data['license'].lower() - - # Ugly backwards compatibility hack - if 'name' in self._data and not isinstance(self._data['name'], dict): - self._data['name'] = {'': self._data['name']} - - self._origdata = copy.deepcopy(self._data) - - for field, default in copy.deepcopy(ProblemConfig._OPTIONAL_CONFIG).items(): - if not field in self._data: - self._data[field] = default - elif isinstance(default, dict) and isinstance(self._data[field], dict): - self._data[field] = dict(list(default.items()) + list(self._data[field].items())) - - val = self._data['validation'].split() - self._data['validation-type'] = val[0] - self._data['validation-params'] = val[1:] - - self._data['grading']['custom_scoring'] = False - for param in self._data['validation-params']: - if param == 'score': - self._data['grading']['custom_scoring'] = True - elif param == 'interactive': - pass - - return self._data - + if self.metadata.data is None: + raise VerifyError(f"Failed to load problem config {self.configfile}") + return self.metadata.data + def __str__(self) -> str: return 'problem configuration' - + def check(self, context: Context) -> bool: if self._check_res is not None: return self._check_res self._check_res = True - + if not os.path.isfile(self.configfile): self.error(f"No config file {self.configfile} found") + + self.metadata.check_config() + + return self._check_res - for field in ProblemConfig._MANDATORY_CONFIG: - if not field in self._data: - self.error(f"Mandatory field '{field}' not provided") - - for field, value in self._origdata.items(): - if field not in ProblemConfig._OPTIONAL_CONFIG.keys() and field not in ProblemConfig._MANDATORY_CONFIG: - self.warning(f"Unknown field '{field}' provided in problem.yaml") - - for field, value in self._data.items(): - if value is None: - self.error(f"Field '{field}' provided in problem.yaml but is empty") - self._data[field] = ProblemConfig._OPTIONAL_CONFIG.get(field, '') - - # Check type - if not self._data['type'] in ['pass-fail', 'scoring']: - self.error(f"Invalid value '{self._data['type']}' for type") - - # Check rights_owner - if self._data['license'] == 'public domain': - if self._data['rights_owner'].strip() != '': - self.error('Can not have a rights_owner for a problem in public domain') - elif self._data['license'] != 'unknown': - if self._data['rights_owner'].strip() == '': - self.error('No author, source or rights_owner provided') - - # Check source_url - if (self._data['source_url'].strip() != '' and - self._data['source'].strip() == ''): - self.error('Can not provide source_url without also providing source') - - # Check license - if not self._data['license'] in ProblemConfig._VALID_LICENSES: - self.error(f"Invalid value for license: {self._data['license']}.\n Valid licenses are {ProblemConfig._VALID_LICENSES}") - elif self._data['license'] == 'unknown': - self.warning("License is 'unknown'") - - if self._data['grading']['show_test_data_groups'] not in [True, False]: - self.error(f"Invalid value for grading.show_test_data_groups: {self._data['grading']['show_test_data_groups']}") - elif self._data['grading']['show_test_data_groups'] and self._data['type'] == 'pass-fail': - self.error("Showing test data groups is only supported for scoring problems, this is a pass-fail problem") - if self._data['type'] != 'pass-fail' and self.problem.get(ProblemTestCases)['root_group'].has_custom_groups() and 'show_test_data_groups' not in self._origdata.get('grading', {}): - self.warning("Problem has custom testcase groups, but does not specify a value for grading.show_test_data_groups; defaulting to false") - - if 'on_reject' in self._data['grading']: - if self._data['type'] == 'pass-fail' and self._data['grading']['on_reject'] == 'grade': - self.error(f"Invalid on_reject policy '{self._data['grading']['on_reject']}' for problem type '{self._data['type']}'") - if not self._data['grading']['on_reject'] in ['first_error', 'worst_error', 'grade']: - self.error(f"Invalid value '{self._data['grading']['on_reject']}' for on_reject policy") - - if self._data['grading']['objective'] not in ['min', 'max']: - self.error(f"Invalid value '{self._data['grading']['objective']}' for objective") - - for deprecated_grading_key in ['accept_score', 'reject_score', 'range', 'on_reject']: - if deprecated_grading_key in self._data['grading']: - self.warning(f"Grading key '{deprecated_grading_key}' is deprecated in problem.yaml, use '{deprecated_grading_key}' in testdata.yaml instead") - - if not self._data['validation-type'] in ['default', 'custom']: - self.error(f"Invalid value '{self._data['validation']}' for validation, first word must be 'default' or 'custom'") - - if self._data['validation-type'] == 'default' and len(self._data['validation-params']) > 0: - self.error(f"Invalid value '{self._data['validation']}' for validation") - - if self._data['validation-type'] == 'custom': - for param in self._data['validation-params']: - if param not in['score', 'interactive']: - self.error(f"Invalid parameter '{param}' for custom validation") - - # Check limits - if not isinstance(self._data['limits'], dict): - self.error('Limits key in problem.yaml must specify a dict') - self._data['limits'] = ProblemConfig._OPTIONAL_CONFIG['limits'] - - # Some things not yet implemented - if self._data['libraries'] != '': - self.error("Libraries not yet supported") +class ProblemConfigLegacy(ProblemConfig): + SPECIFICATION_FILE_NAME = 'legacy_config_specification.yaml' + def check(self, context): + super().check(context) + if (self.problem.get(ProblemConfig)['type'] != 'pass-fail' and + self.problem.get(ProblemTestCases)['root_group'].has_custom_groups() and + 'show_test_data_groups' not in self.problem.get(ProblemConfig)['original_data'].get('grading', {})): + self.warning("Problem has custom testcase groups, but does not specify a value for grading.show_test_data_groups; defaulting to false") return self._check_res +class ProblemConfig2023_07(ProblemConfig): + SPECIFICATION_FILE_NAME = '2023-07_config_specification.yaml' + class ProblemTestCases(ProblemPart): PART_NAME = 'testdata' @@ -947,7 +862,7 @@ def setup(self): self.testcase_by_infile = {} return { 'root_group': TestCaseGroup(self.problem, self.PART_NAME), - 'is_interactive': 'interactive' in self.problem.get(ProblemConfig)['validation-params'], + 'is_interactive': self.problem.get(ProblemConfig)['validation']['interactive'], 'is_scoring': self.problem.get(ProblemConfig)['type'] == 'scoring' } @@ -1268,12 +1183,12 @@ def check(self, context: Context) -> bool: if isinstance(v, run.SourceCode) and v.language.lang_id not in recommended_output_validator_languages: self.warning('output validator language %s is not recommended' % v.language.name) - if self.problem.get(ProblemConfig)['validation'] == 'default' and self._validators: + if self.problem.get(ProblemConfig)['validation']['type'] == 'default' and self._validators: self.error('There are validator programs but problem.yaml has validation = "default"') - elif self.problem.get(ProblemConfig)['validation'] != 'default' and not self._validators: + elif self.problem.get(ProblemConfig)['validation']['type'] != 'default' and not self._validators: self.error('problem.yaml specifies custom validator but no validator programs found') - if self.problem.get(ProblemConfig)['validation'] == 'default' and self._default_validator is None: + if self.problem.get(ProblemConfig)['validation']['type'] == 'default' and self._default_validator is None: self.error('Unable to locate default validator') for val in self._validators[:]: @@ -1363,7 +1278,7 @@ def _parse_validator_results(self, val, status: int, feedbackdir, testcase: Test def _actual_validators(self) -> list: vals = self._validators - if self.problem.get(ProblemConfig)['validation'] == 'default': + if self.problem.get(ProblemConfig)['validation']['type'] == 'default': vals = [self._default_validator] return [val for val in vals if val is not None] @@ -1729,7 +1644,7 @@ def check(self, context: Context) -> bool: PROBLEM_FORMATS: dict[str, dict[str, list[Type[ProblemPart]]]] = { 'legacy': { - 'config': [ProblemConfig], + 'config': [ProblemConfigLegacy], 'statement': [ProblemStatementLegacy, Attachments], 'validators': [InputValidators, OutputValidators], 'graders': [Graders], @@ -1737,6 +1652,7 @@ def check(self, context: Context) -> bool: 'submissions': [Submissions], }, '2023-07': { # TODO: Add all the parts + 'config': [ProblemConfig2023_07], 'statement': [ProblemStatement2023_07, Attachments], } } @@ -1773,6 +1689,8 @@ def getProblemPart(self, part: Type[_ProblemPartT]) -> _ProblemPartT: return self._classes[part.PART_NAME] # type: ignore def __enter__(self) -> Problem: + ProblemAspect.errors = 0 + ProblemAspect.warnings = 0 self.tmpdir = tempfile.mkdtemp(prefix=f'verify-{self.shortname}-') if not os.path.isdir(self.probdir): self.error(f"Problem directory '{self.probdir}' not found") @@ -1817,8 +1735,6 @@ def check(self, args: argparse.Namespace) -> tuple[int, int]: if self.shortname is None: return 1, 0 - ProblemAspect.errors = 0 - ProblemAspect.warnings = 0 ProblemAspect.bail_on_error = args.bail_on_error ProblemAspect.consider_warnings_errors = args.werror