fix(grounding): auto-rescue hallucinated invocation_id + list real ids in error
First full-case run (runs/2026-05-20T20-15-04/) produced 83 GroundingError rejections, almost all from a single failure mode: LLM cites a plausible- looking inv-XXXXXXXX that doesn't exist, while the fact's value is in fact present verbatim in one of its real tool outputs. The agent knew which tool it read from, it just mis-typed the citation id. Two-layer fix in evidence_graph.validate_fact_grounding: Layer A (silent heal): when the cited inv-id misses, search the same agent / task's invocations for one whose output contains the value (strict or normalised substring). If exactly one matches, rewrite fact.invocation_id in place and accept. Multi-match is NOT auto- rescued — the candidate ids go back to the LLM so it picks deliberately. Layer B (informative retry): GroundingError now appends the agent's recent invocation ids and a brief tool-call summary, so the LLM has the real ids in front of it for the next attempt rather than fabricating again from memory. Both layers preserve the design invariant: the fact's value must still be present in a real tool output — nothing new can land grounded that wasn't already verifiable. Cross-agent / cross-task isolation is also preserved (rescue candidates filtered on agent + task_id). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1779,6 +1779,103 @@ class TestGroundingGateway:
|
||||
)
|
||||
assert "not found in invocation log" in str(exc.value)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_rescue_single_match_rewrites_invocation_id(self, graph):
|
||||
"""Layer A: agent cites a bogus inv id but the fact value is uniquely
|
||||
present in one of its real invocations → silently heal the citation
|
||||
and accept the fact.
|
||||
"""
|
||||
real_inv = await graph.record_tool_invocation(
|
||||
tool="sqlite_query",
|
||||
args={"db": "AddressBook.sqlitedb"},
|
||||
output="Hogan | +852 5497 4406 | whoishogan@gmail.com",
|
||||
)
|
||||
bogus_id = "inv-deadbeef"
|
||||
facts = [{"type": "identifier", "value": "+852 5497 4406", "invocation_id": bogus_id}]
|
||||
pid, _ = await graph.add_phenomenon(
|
||||
source_agent="fs", category="identity",
|
||||
title="phone for Hogan",
|
||||
verified_facts=facts,
|
||||
source_tool="sqlite_query",
|
||||
)
|
||||
ph = graph.phenomena[pid]
|
||||
assert ph.verified_facts[0]["invocation_id"] == real_inv
|
||||
assert ph.verified_facts[0]["invocation_id"] != bogus_id
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_rescue_skips_when_value_matches_multiple_invocations(self, graph):
|
||||
"""Layer A safety: ambiguous match (value present in >1 invocation)
|
||||
is NOT silently rewritten — the LLM gets the candidate list back so
|
||||
it picks the right id on retry.
|
||||
"""
|
||||
inv_a = await graph.record_tool_invocation(
|
||||
tool="list_directory", args={"dir": "1"},
|
||||
output="d/d 33-128-1: secret.txt\nfound on disk",
|
||||
)
|
||||
inv_b = await graph.record_tool_invocation(
|
||||
tool="list_directory", args={"dir": "2"},
|
||||
output="d/d 99-128-1: vault.txt\nfound on disk",
|
||||
)
|
||||
with pytest.raises(GroundingError) as exc:
|
||||
await graph.add_phenomenon(
|
||||
source_agent="fs", category="filesystem", title="dup",
|
||||
verified_facts=[
|
||||
{"type": "raw", "value": "found on disk", "invocation_id": "inv-nope"},
|
||||
],
|
||||
source_tool="list_directory",
|
||||
)
|
||||
msg = str(exc.value)
|
||||
assert inv_a in msg
|
||||
assert inv_b in msg
|
||||
assert "2 of your invocations" in msg
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_grounding_error_lists_recent_invocations(self, graph):
|
||||
"""Layer B: on rejection, the GroundingError message appends the
|
||||
agent's recent real invocation ids so the LLM can cite a valid one
|
||||
on retry instead of fabricating again.
|
||||
"""
|
||||
inv_one = await graph.record_tool_invocation(
|
||||
tool="fls", args={"offset": 614400}, output="some output A",
|
||||
)
|
||||
inv_two = await graph.record_tool_invocation(
|
||||
tool="icat", args={"inode": "33"}, output="some output B",
|
||||
)
|
||||
with pytest.raises(GroundingError) as exc:
|
||||
await graph.add_phenomenon(
|
||||
source_agent="fs", category="filesystem", title="bogus",
|
||||
verified_facts=[
|
||||
{"type": "raw", "value": "totally absent string",
|
||||
"invocation_id": "inv-bogus"},
|
||||
],
|
||||
source_tool="fls",
|
||||
)
|
||||
msg = str(exc.value)
|
||||
assert "Your recent invocations in this task" in msg
|
||||
assert inv_one in msg
|
||||
assert inv_two in msg
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_rescue_respects_agent_scope(self, graph):
|
||||
"""Layer A invariant: rescue candidates must be from the SAME agent.
|
||||
A value present only in another agent's invocation must NOT trigger
|
||||
auto-rescue across agents.
|
||||
"""
|
||||
graph._current_agent = "registry"
|
||||
await graph.record_tool_invocation(
|
||||
tool="parse_registry_key", args={}, output="REG_VALUE_xyz",
|
||||
)
|
||||
graph._current_agent = "fs"
|
||||
with pytest.raises(GroundingError):
|
||||
await graph.add_phenomenon(
|
||||
source_agent="fs", category="filesystem", title="cross-agent leak",
|
||||
verified_facts=[
|
||||
{"type": "raw", "value": "REG_VALUE_xyz",
|
||||
"invocation_id": "inv-anything"},
|
||||
],
|
||||
source_tool="parse_registry_key",
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_verified_facts_allowed_for_negative_findings(self, graph):
|
||||
# A negative finding ("searched X, found nothing") is permitted —
|
||||
|
||||
Reference in New Issue
Block a user