From 800f3440019caacc6318bb3e581e31892c9bc9a9 Mon Sep 17 00:00:00 2001 From: Matistjati Date: Sat, 6 Dec 2025 23:48:15 +0100 Subject: [PATCH 1/2] Check if there are incorrect submission directories --- problemtools/formatversion.py | 11 +++++++++++ problemtools/verifyproblem.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/problemtools/formatversion.py b/problemtools/formatversion.py index 757746e2..dcca51ca 100644 --- a/problemtools/formatversion.py +++ b/problemtools/formatversion.py @@ -31,6 +31,14 @@ def output_validator_directory(self) -> str: case FormatVersion.V_2023_07: return 'output_validator' + @property + def submission_directories(self) -> list[str]: + match self: + case FormatVersion.LEGACY: + return ['accepted', 'partially_accepted', 'wrong_answer', 'time_limit_exceeded', 'run_time_error'] + case FormatVersion.V_2023_07: + return ['accepted', 'rejected', 'wrong_answer', 'time_limit_exceeded', 'run_time_error', 'brute_force'] + # Support 2023-07 and 2023-07-draft strings. # This method should be replaced with an alias once we require python 3.13 @classmethod @@ -40,6 +48,9 @@ def _missing_(cls, value): return None +ALL_FORMAT_VERSIONS = [FormatVersion.LEGACY, FormatVersion.V_2023_07] + + def get_format_version(problem_root: Path) -> FormatVersion: """Loads the version from the problem in problem_root""" with open(problem_root / 'problem.yaml') as f: diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 2538f0c2..71687f22 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -23,6 +23,7 @@ import random import traceback import uuid +import difflib from pathlib import Path import colorlog @@ -35,7 +36,7 @@ from . import problem2pdf from . import run from . import statement_util -from .formatversion import FormatVersion, get_format_version +from .formatversion import FormatVersion, get_format_version, ALL_FORMAT_VERSIONS from .version import add_version_arg from abc import ABC @@ -2044,6 +2045,7 @@ def check(self) -> tuple[int, int]: self._check_symlinks() self._check_file_and_directory_names() + self._check_submission_directory_names() run.limit.check_limit_capabilities(self) @@ -2067,6 +2069,37 @@ def check(self) -> tuple[int, int]: context.wait_for_background_work() return self.errors, self.warnings + def _check_submission_directory_names(self): + """Heuristically check if submissions contain any directories that will be ignored because of typos or format mismatches""" + submission_directories = [p.name for p in (Path(self.probdir) / 'submissions').glob('*') if p.is_dir()] + if len(submission_directories) == 0: + return + + def most_similar(present_dir: str, format_version: FormatVersion): + similarities = [ + (spec_dir, difflib.SequenceMatcher(None, present_dir, spec_dir).ratio()) + for spec_dir in format_version.submission_directories + ] + return max(similarities, key=lambda x: x[1]) + + for present_dir in submission_directories: + most_similar_dir, max_similarity = most_similar(present_dir, self.format) + + if max_similarity == 1: + # Exact match, no typo + continue + + if 0.75 <= max_similarity: + self.warning(f'Potential typo: directory submissions/{present_dir} is similar to {most_similar_dir}') + else: + for other_version in [v for v in ALL_FORMAT_VERSIONS if v != self.format]: + _, max_similarity = most_similar(present_dir, other_version) + if max_similarity == 1: + self.warning( + f'Directory submissions/{present_dir} is not part of format version {self.format}, but part of {other_version}' + ) + break + def _check_symlinks(self): """Check that all symlinks point to something existing within the problem package""" probdir = os.path.realpath(self.probdir) From 7938046260fd27385c0fb4d2e9a6ead0d4bf0d78 Mon Sep 17 00:00:00 2001 From: Matistjati Date: Sun, 7 Dec 2025 15:48:09 +0100 Subject: [PATCH 2/2] Add TODO to parse submissions.yaml --- problemtools/formatversion.py | 5 ++--- problemtools/verifyproblem.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/problemtools/formatversion.py b/problemtools/formatversion.py index dcca51ca..c13cae95 100644 --- a/problemtools/formatversion.py +++ b/problemtools/formatversion.py @@ -37,6 +37,8 @@ def submission_directories(self) -> list[str]: case FormatVersion.LEGACY: return ['accepted', 'partially_accepted', 'wrong_answer', 'time_limit_exceeded', 'run_time_error'] case FormatVersion.V_2023_07: + # TODO: parse submissions.yaml if applicable, since + # 2023-07 and later formats support adding more submission directories return ['accepted', 'rejected', 'wrong_answer', 'time_limit_exceeded', 'run_time_error', 'brute_force'] # Support 2023-07 and 2023-07-draft strings. @@ -48,9 +50,6 @@ def _missing_(cls, value): return None -ALL_FORMAT_VERSIONS = [FormatVersion.LEGACY, FormatVersion.V_2023_07] - - def get_format_version(problem_root: Path) -> FormatVersion: """Loads the version from the problem in problem_root""" with open(problem_root / 'problem.yaml') as f: diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 71687f22..295fb465 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -36,7 +36,7 @@ from . import problem2pdf from . import run from . import statement_util -from .formatversion import FormatVersion, get_format_version, ALL_FORMAT_VERSIONS +from .formatversion import FormatVersion, get_format_version from .version import add_version_arg from abc import ABC @@ -2092,7 +2092,7 @@ def most_similar(present_dir: str, format_version: FormatVersion): if 0.75 <= max_similarity: self.warning(f'Potential typo: directory submissions/{present_dir} is similar to {most_similar_dir}') else: - for other_version in [v for v in ALL_FORMAT_VERSIONS if v != self.format]: + for other_version in [v for v in FormatVersion if v != self.format]: _, max_similarity = most_similar(present_dir, other_version) if max_similarity == 1: self.warning(