Check if there are incorrect submission directories

problemtools/formatversion.py

@@ -31,6 +31,16 @@ def output_validator_directory(self) -> str:
             case FormatVersion.V_2023_07:
                 return 'output_validator'
 
+    @property
+    def submission_directories(self) -> list[str]:
+        match self:
+            case FormatVersion.LEGACY:
+                return ['accepted', 'partially_accepted', 'wrong_answer', 'time_limit_exceeded', 'run_time_error']
+            case FormatVersion.V_2023_07:
+                # TODO: parse submissions.yaml if applicable, since
+                # 2023-07 and later formats support adding more submission directories
+                return ['accepted', 'rejected', 'wrong_answer', 'time_limit_exceeded', 'run_time_error', 'brute_force']
+
     # Support 2023-07 and 2023-07-draft strings.
     # This method should be replaced with an alias once we require python 3.13
     @classmethod

problemtools/verifyproblem.py

@@ -23,6 +23,7 @@
 import random
 import traceback
 import uuid
+import difflib
 from pathlib import Path
 
 import colorlog
@@ -2044,6 +2045,7 @@ def check(self) -> tuple[int, int]:
 
             self._check_symlinks()
             self._check_file_and_directory_names()
+            self._check_submission_directory_names()
 
             run.limit.check_limit_capabilities(self)
 
@@ -2067,6 +2069,37 @@ def check(self) -> tuple[int, int]:
             context.wait_for_background_work()
         return self.errors, self.warnings
 
+    def _check_submission_directory_names(self):
+        """Heuristically check if submissions contain any directories that will be ignored because of typos or format mismatches"""
+        submission_directories = [p.name for p in (Path(self.probdir) / 'submissions').glob('*') if p.is_dir()]
+        if len(submission_directories) == 0:
+            return
+
+        def most_similar(present_dir: str, format_version: FormatVersion):
+            similarities = [
+                (spec_dir, difflib.SequenceMatcher(None, present_dir, spec_dir).ratio())
+                for spec_dir in format_version.submission_directories
+            ]
+            return max(similarities, key=lambda x: x[1])
+
+        for present_dir in submission_directories:
+            most_similar_dir, max_similarity = most_similar(present_dir, self.format)
+
+            if max_similarity == 1:
+                # Exact match, no typo
+                continue
+
+            if 0.75 <= max_similarity:
+                self.warning(f'Potential typo: directory submissions/{present_dir} is similar to {most_similar_dir}')
+            else:
+                for other_version in [v for v in FormatVersion if v != self.format]:
+                    _, max_similarity = most_similar(present_dir, other_version)
+                    if max_similarity == 1:
+                        self.warning(
+                            f'Directory submissions/{present_dir} is not part of format version {self.format}, but part of {other_version}'
+                        )
+                        break
+
     def _check_symlinks(self):
         """Check that all symlinks point to something existing within the problem package"""
         probdir = os.path.realpath(self.probdir)