fix(grounding): auto-rescue hallucinated invocation_id + list real ids in error

First full-case run (runs/2026-05-20T20-15-04/) produced 83 GroundingError
rejections, almost all from a single failure mode: LLM cites a plausible-
looking inv-XXXXXXXX that doesn't exist, while the fact's value is in fact
present verbatim in one of its real tool outputs. The agent knew which
tool it read from, it just mis-typed the citation id.

Two-layer fix in evidence_graph.validate_fact_grounding:

  Layer A (silent heal): when the cited inv-id misses, search the same
  agent / task's invocations for one whose output contains the value
  (strict or normalised substring). If exactly one matches, rewrite
  fact.invocation_id in place and accept. Multi-match is NOT auto-
  rescued — the candidate ids go back to the LLM so it picks deliberately.

  Layer B (informative retry): GroundingError now appends the agent's
  recent invocation ids and a brief tool-call summary, so the LLM has
  the real ids in front of it for the next attempt rather than
  fabricating again from memory.

Both layers preserve the design invariant: the fact's value must still
be present in a real tool output — nothing new can land grounded that
wasn't already verifiable. Cross-agent / cross-task isolation is also
preserved (rescue candidates filtered on agent + task_id).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
BattleTag
2026-05-21 02:14:20 -10:00
parent 81ade8f7ac
commit 6b485b98f7
2 changed files with 179 additions and 6 deletions

View File

@@ -1779,6 +1779,103 @@ class TestGroundingGateway:
)
assert "not found in invocation log" in str(exc.value)
@pytest.mark.asyncio
async def test_auto_rescue_single_match_rewrites_invocation_id(self, graph):
"""Layer A: agent cites a bogus inv id but the fact value is uniquely
present in one of its real invocations → silently heal the citation
and accept the fact.
"""
real_inv = await graph.record_tool_invocation(
tool="sqlite_query",
args={"db": "AddressBook.sqlitedb"},
output="Hogan | +852 5497 4406 | whoishogan@gmail.com",
)
bogus_id = "inv-deadbeef"
facts = [{"type": "identifier", "value": "+852 5497 4406", "invocation_id": bogus_id}]
pid, _ = await graph.add_phenomenon(
source_agent="fs", category="identity",
title="phone for Hogan",
verified_facts=facts,
source_tool="sqlite_query",
)
ph = graph.phenomena[pid]
assert ph.verified_facts[0]["invocation_id"] == real_inv
assert ph.verified_facts[0]["invocation_id"] != bogus_id
@pytest.mark.asyncio
async def test_auto_rescue_skips_when_value_matches_multiple_invocations(self, graph):
"""Layer A safety: ambiguous match (value present in >1 invocation)
is NOT silently rewritten — the LLM gets the candidate list back so
it picks the right id on retry.
"""
inv_a = await graph.record_tool_invocation(
tool="list_directory", args={"dir": "1"},
output="d/d 33-128-1: secret.txt\nfound on disk",
)
inv_b = await graph.record_tool_invocation(
tool="list_directory", args={"dir": "2"},
output="d/d 99-128-1: vault.txt\nfound on disk",
)
with pytest.raises(GroundingError) as exc:
await graph.add_phenomenon(
source_agent="fs", category="filesystem", title="dup",
verified_facts=[
{"type": "raw", "value": "found on disk", "invocation_id": "inv-nope"},
],
source_tool="list_directory",
)
msg = str(exc.value)
assert inv_a in msg
assert inv_b in msg
assert "2 of your invocations" in msg
@pytest.mark.asyncio
async def test_grounding_error_lists_recent_invocations(self, graph):
"""Layer B: on rejection, the GroundingError message appends the
agent's recent real invocation ids so the LLM can cite a valid one
on retry instead of fabricating again.
"""
inv_one = await graph.record_tool_invocation(
tool="fls", args={"offset": 614400}, output="some output A",
)
inv_two = await graph.record_tool_invocation(
tool="icat", args={"inode": "33"}, output="some output B",
)
with pytest.raises(GroundingError) as exc:
await graph.add_phenomenon(
source_agent="fs", category="filesystem", title="bogus",
verified_facts=[
{"type": "raw", "value": "totally absent string",
"invocation_id": "inv-bogus"},
],
source_tool="fls",
)
msg = str(exc.value)
assert "Your recent invocations in this task" in msg
assert inv_one in msg
assert inv_two in msg
@pytest.mark.asyncio
async def test_auto_rescue_respects_agent_scope(self, graph):
"""Layer A invariant: rescue candidates must be from the SAME agent.
A value present only in another agent's invocation must NOT trigger
auto-rescue across agents.
"""
graph._current_agent = "registry"
await graph.record_tool_invocation(
tool="parse_registry_key", args={}, output="REG_VALUE_xyz",
)
graph._current_agent = "fs"
with pytest.raises(GroundingError):
await graph.add_phenomenon(
source_agent="fs", category="filesystem", title="cross-agent leak",
verified_facts=[
{"type": "raw", "value": "REG_VALUE_xyz",
"invocation_id": "inv-anything"},
],
source_tool="parse_registry_key",
)
@pytest.mark.asyncio
async def test_empty_verified_facts_allowed_for_negative_findings(self, graph):
# A negative finding ("searched X, found nothing") is permitted —