Merge pull request #1383 from datajoint/fix/jobs-semantic-check-1379

fix: Disable semantic_check for job table subtraction in refresh()

Author: dimitri-yatsenko

src/datajoint/jobs.py

@@ -13,7 +13,7 @@
 import platform
 import subprocess
 
-from .condition import AndList
+from .condition import AndList, Not
 from .errors import DataJointError, DuplicateError
 from .heading import Heading
 from .table import Table
@@ -370,8 +370,6 @@ def refresh(
 
         # Keys that need jobs: in key_source, not in target, not in jobs
         # Disable semantic_check for Job table (self) because its attributes may not have matching lineage
-        from .condition import Not
-
         new_keys = (key_source - self._target).restrict(Not(self), semantic_check=False).proj()
         new_key_list = new_keys.keys()
 
@@ -395,8 +393,10 @@ def refresh(
         # 2. Re-pend success jobs if keep_completed=True
         if config.jobs.keep_completed:
             # Success jobs whose keys are in key_source but not in target
-            # Disable semantic_check for Job table operations
-            success_to_repend = self.completed.restrict(key_source, semantic_check=False) - self._target
+            # Disable semantic_check for Job table operations (job table PK has different lineage than target)
+            success_to_repend = self.completed.restrict(key_source, semantic_check=False).restrict(
+                Not(self._target), semantic_check=False
+            )
             repend_keys = success_to_repend.keys()
             for key in repend_keys:
                 (self & key).delete_quick()

tests/integration/test_jobs.py

@@ -158,3 +158,52 @@ def test_long_error_stack(clean_jobs, subject, experiment):
     experiment.jobs.error(key, "error message", long_error_stack)
     error_stack = experiment.jobs.errors.fetch1("error_stack")
     assert error_stack == long_error_stack, "error stacks do not agree"
+
+
+def test_populate_reserve_jobs_with_keep_completed(clean_jobs, subject, experiment):
+    """Test populate(reserve_jobs=True) with keep_completed=True.
+
+    Regression test for https://github.com/datajoint/datajoint-python/issues/1379
+    The bug was that the `-` operator in jobs.refresh() didn't pass semantic_check=False,
+    causing a DataJointError about different lineages when keep_completed=True.
+    """
+    # Clear experiment data to ensure there's work to do
+    experiment.delete()
+
+    with dj.config.override(jobs={"keep_completed": True, "add_job_metadata": True}):
+        # Should not raise DataJointError about semantic matching
+        experiment.populate(reserve_jobs=True)
+
+        # Verify jobs completed successfully
+        assert len(experiment) > 0, "No data was populated"
+        assert len(experiment.jobs.errors) == 0, "Unexpected errors during populate"
+
+        # With keep_completed=True, completed jobs should be retained
+        assert len(experiment.jobs.completed) > 0, "Completed jobs not retained"
+
+
+def test_jobs_refresh_with_keep_completed(clean_jobs, subject, experiment):
+    """Test that jobs.refresh() works with keep_completed=True.
+
+    Regression test for https://github.com/datajoint/datajoint-python/issues/1379
+    """
+    # Clear experiment data and jobs
+    experiment.delete()
+    experiment.jobs.delete()
+
+    with dj.config.override(jobs={"keep_completed": True, "add_job_metadata": True}):
+        # Refresh should create pending jobs without semantic matching error
+        experiment.jobs.refresh()
+        pending_before = len(experiment.jobs.pending)
+        assert pending_before > 0, "No pending jobs created"
+
+        # Manually reserve and complete a job
+        key = experiment.jobs.pending.keys(limit=1)[0]
+        experiment.jobs.reserve(key)
+        experiment.jobs.complete(key)
+
+        # Job should now be completed
+        assert len(experiment.jobs.completed) == 1, "Job not marked as completed"
+
+        # Calling refresh again should not raise semantic matching error
+        experiment.jobs.refresh()  # This was failing before the fix