Fix populate antijoin to use .proj() for correct pending key computation

The antijoin that computes pending keys (`key_source - self` in
`_populate_direct`, `key_source - self._target` in `jobs.refresh`, and
`todo - self` in `progress`) did not project the target table to its
primary key before the subtraction. When the target table has secondary
(non-PK) attributes, the antijoin fails to match on primary key alone
and returns all keys instead of just the unpopulated ones.

This caused:
- `populate(reserve_jobs=False)`: all key_source entries were iterated
  instead of just pending ones (mitigated by `if key in self:` check
  inside `_populate1`, but wasted time on large tables)
- `populate(reserve_jobs=True)`: `jobs.refresh()` inserted all keys into
  the jobs table as 'pending', not just truly pending ones. Workers then
  wasted their `max_calls` budget processing already-completed entries
  before reaching any real work.
- `progress()`: reported incorrect remaining counts in some cases

Fix: add `.proj()` to the target side of all three antijoins so the
subtraction matches on primary key only, consistent with how DataJoint
antijoins are meant to work.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: 2 people

src/datajoint/autopopulate.py

@@ -403,7 +403,7 @@ def _populate_direct(
         """
         from tqdm import tqdm
 
-        keys = (self._jobs_to_do(restrictions) - self).keys()
+        keys = (self._jobs_to_do(restrictions) - self.proj()).keys()
 
         logger.debug("Found %d keys to populate" % len(keys))
 
@@ -701,7 +701,7 @@ def progress(self, *restrictions: Any, display: bool = False) -> tuple[int, int]
         if not common_attrs:
             # No common attributes - fall back to two-query method
             total = len(todo)
-            remaining = len(todo - self)
+            remaining = len(todo - self.proj())
         else:
             # Build a single query that computes both total and remaining
             # Using LEFT JOIN with COUNT(DISTINCT) to handle 1:many relationships

src/datajoint/jobs.py

@@ -370,7 +370,7 @@ def refresh(
 
         # Keys that need jobs: in key_source, not in target, not in jobs
         # Disable semantic_check for Job table (self) because its attributes may not have matching lineage
-        new_keys = (key_source - self._target).restrict(Not(self), semantic_check=False).proj()
+        new_keys = (key_source - self._target.proj()).restrict(Not(self), semantic_check=False).proj()
         new_key_list = new_keys.keys()
 
         if new_key_list:

tests/integration/test_autopopulate.py

@@ -112,6 +112,79 @@ def test_allow_insert(clean_autopopulate, subject, experiment):
         experiment.insert1(key)
 
 
+def test_populate_antijoin_with_secondary_attrs(clean_autopopulate, subject, experiment):
+    """Test that populate correctly computes pending keys via antijoin.
+
+    Regression test for a bug where `key_source - self` returned all keys
+    instead of just unpopulated ones when the target table has secondary
+    attributes. The antijoin must match on primary key only, ignoring
+    secondary attributes. Without `.proj()`, the antijoin could fail to
+    exclude already-populated keys.
+
+    This affected both direct mode (reserve_jobs=False) and distributed mode
+    (reserve_jobs=True), causing workers to waste time re-checking already
+    completed entries.
+    """
+    assert subject, "root tables are empty"
+    assert not experiment, "table already filled?"
+
+    total_keys = len(experiment.key_source)
+    assert total_keys > 0
+
+    # Partially populate (only 2 entries)
+    experiment.populate(max_calls=2)
+    assert len(experiment) == 2
+
+    # The critical test: key_source - target must return only unpopulated keys.
+    # Before the fix, this returned all keys (== total_keys) because the
+    # antijoin failed to match on PK when secondary attributes were present.
+    pending = experiment.key_source - experiment
+    assert len(pending) == total_keys - 2, (
+        f"Antijoin returned {len(pending)} pending keys, expected {total_keys - 2}. "
+        f"key_source - target may not be matching on primary key only."
+    )
+
+    # Also verify progress() reports correct counts
+    remaining, total = experiment.progress()
+    assert total == total_keys
+    assert remaining == total_keys - 2
+
+    # Populate the rest and verify antijoin returns 0
+    experiment.populate()
+    pending_after = experiment.key_source - experiment
+    assert len(pending_after) == 0, (
+        f"Antijoin returned {len(pending_after)} pending keys after full populate, expected 0."
+    )
+
+
+def test_populate_distributed_antijoin(clean_autopopulate, subject, experiment):
+    """Test that reserve_jobs=True correctly identifies pending keys.
+
+    When using distributed mode, jobs.refresh() must only insert truly pending
+    keys into the jobs table, not already-completed ones. This verifies the
+    antijoin in jobs.refresh() works correctly with secondary attributes.
+    """
+    assert subject, "root tables are empty"
+    assert not experiment, "table already filled?"
+
+    total_keys = len(experiment.key_source)
+
+    # Partially populate
+    experiment.populate(max_calls=2)
+    assert len(experiment) == 2
+
+    # Refresh jobs — should only create entries for unpopulated keys
+    experiment.jobs.refresh(delay=-1)
+    pending_jobs = len(experiment.jobs.pending)
+    assert pending_jobs == total_keys - 2, (
+        f"jobs.refresh() created {pending_jobs} pending jobs, expected {total_keys - 2}. "
+        f"The antijoin in refresh() may not be excluding already-completed keys."
+    )
+
+    # Clean up
+    experiment.jobs.delete_quick()
+
+
 def test_load_dependencies(prefix, connection_test):
     schema = dj.Schema(f"{prefix}_load_dependencies_populate", connection=connection_test)