BattleTag/MASForensic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(graph): harmonic damping for repeated same-edge_type evidence

Browse Source 
First full-case run (runs/2026-05-20T20-15-04/) produced hypotheses
with log_odds +31 (8 direct_evidence + 15 supports). That's the
naive-Bayes independence assumption breaking down: 15 different
phenomena all "supporting" the same hypothesis from one source are
not 15 independent pieces of evidence, they're highly correlated.
DESIGN.md §4.5 last bullet flagged this as a "未实施旋钮" — this
commit implements it.

Rule: the k-th edge of a given (hyp_id, edge_type) contributes
log_lr_base / k instead of log_lr_base. Cumulative is harmonic
sum H_N, bounded by ~ ln N. Single-edge hypotheses unaffected
(k=1 → /1 → no change). Replaying the 2026-05-20 graph's 108
edges under the new rule pulls the top hypothesis from +31.0 →
+8.75; the smallest active hypothesis from +4.0 → +2.08.

Also adds rank + log_lr_base to confidence_log entries so the
math is auditable from the persisted graph.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
BattleTag
2026-05-21 02:16:37 -10:00
parent f04ccd4bc7
commit 8020c24776
2 changed files with 110 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
tests/test_optimizations.py
View File

@@ -2020,8 +2020,9 @@ class TestLogOddsConfidence:
# All three orderings must agree exactly.
assert confs[0] == pytest.approx(confs[1])
assert confs[1] == pytest.approx(confs[2])
# And the value should be 1 + 1 − 0.5 = 1.5 → sigmoid ≈ 0.9694
assert confs[0] == pytest.approx(0.9694, abs=1e-3)
# With harmonic damping: 2 supports → 1.0 + 0.5 = +1.5, 1 weakens
# → −0.5, net log_odds = +1.0 → sigmoid 1/(1+10^-1) ≈ 0.9091.
assert confs[0] == pytest.approx(0.9091, abs=1e-3)
@pytest.mark.asyncio
async def test_each_edge_type_calibrated(self, graph):
@@ -2082,17 +2083,91 @@ class TestLogOddsConfidence:
assert sum(1 for e in graph.edges if e.edge_type == "supports") == first_edges
@pytest.mark.asyncio
async def test_independent_evidence_accumulates(self, graph):
"""Distinct phenomena with same edge_type DO accumulate (independent)."""
async def test_repeated_same_direction_evidence_dampens_harmonically(self, graph):
"""Distinct phenomena with same edge_type accumulate WITH harmonic
damping (post 2026-05-20 full-case-run fix). The k-th edge of a
given (hyp, edge_type) contributes log_lr_base/k; after 3 supports
edges the total is 1.0 · (1 + 1/2 + 1/3) ≈ 1.833, not 3.0. This
formalises the breakdown of the naive-Bayes independence
assumption when multiple agents pile on the same finding.
"""
hid = await graph.add_hypothesis("h", "d")
for i in range(3):
pid, _ = await graph.add_phenomenon(
"fs", "filesystem", f"ph {i}", f"d {i}", source_tool="t",
)
await graph.update_hypothesis_confidence(hid, pid, "supports", "")
# 3 × +1.0 = +3.0 log_odds → conf ≈ 0.999
assert graph.hypotheses[hid].log_odds == pytest.approx(3.0)
assert graph.hypotheses[hid].confidence > 0.99
# 1.0 · H_3 = 1.0 + 0.5 + 0.3333... = 1.8333
expected = 1.0 + 0.5 + (1.0 / 3.0)
assert graph.hypotheses[hid].log_odds == pytest.approx(expected, abs=1e-6)
# Still above the 0.8 supported threshold (good — 3 independent
# observations remain strong evidence) but well below the runaway
# confidence pre-fix produced.
assert graph.hypotheses[hid].status == "supported"
@pytest.mark.asyncio
async def test_first_edge_undamped(self, graph):
"""Damping is rank-1 → /1 → no change. A hypothesis with a single
edge must contribute the full calibrated log_lr value (otherwise the
whole calibration table would be off by a factor)."""
hid = await graph.add_hypothesis("h", "d")
pid, _ = await graph.add_phenomenon(
"fs", "filesystem", "lone", "d", source_tool="t",
)
await graph.update_hypothesis_confidence(hid, pid, "direct_evidence", "")
assert graph.hypotheses[hid].log_odds == pytest.approx(2.0)
@pytest.mark.asyncio
async def test_damping_independent_per_edge_type(self, graph):
"""Damping rank is keyed per (hyp, edge_type) — supports' counter
does NOT advance direct_evidence's counter."""
hid = await graph.add_hypothesis("h", "d")
for i in range(5):
pid, _ = await graph.add_phenomenon(
"fs", "filesystem", f"s{i}", "d", source_tool="t",
)
await graph.update_hypothesis_confidence(hid, pid, "supports", "")
log_after_supports = graph.hypotheses[hid].log_odds
# H_5 = 1 + 1/2 + 1/3 + 1/4 + 1/5 = 2.2833
assert log_after_supports == pytest.approx(2.2833, abs=1e-3)
pid_de, _ = await graph.add_phenomenon(
"fs", "filesystem", "de", "d", source_tool="t",
)
await graph.update_hypothesis_confidence(hid, pid_de, "direct_evidence", "")
delta = graph.hypotheses[hid].log_odds - log_after_supports
assert delta == pytest.approx(2.0, abs=1e-6)
@pytest.mark.asyncio
async def test_damping_independent_per_hypothesis(self, graph):
"""The rank counter is per (hyp, edge_type), so two hypotheses
each receiving 'supports' edges accumulate independently."""
h1 = await graph.add_hypothesis("h1", "d")
h2 = await graph.add_hypothesis("h2", "d")
for i in range(3):
pid, _ = await graph.add_phenomenon(
"fs", "filesystem", f"to-h1 {i}", "d", source_tool="t",
)
await graph.update_hypothesis_confidence(h1, pid, "supports", "")
pid_h2, _ = await graph.add_phenomenon(
"fs", "filesystem", "to-h2", "d", source_tool="t",
)
await graph.update_hypothesis_confidence(h2, pid_h2, "supports", "")
assert graph.hypotheses[h2].log_odds == pytest.approx(1.0)
@pytest.mark.asyncio
async def test_damping_rank_persists_in_confidence_log(self, graph):
"""The rank used for damping must be recorded so the math is
auditable from the persisted graph (no need to recompute)."""
hid = await graph.add_hypothesis("h", "d")
for i in range(2):
pid, _ = await graph.add_phenomenon(
"fs", "filesystem", f"p {i}", "d", source_tool="t",
)
await graph.update_hypothesis_confidence(hid, pid, "supports", "")
entries = graph.hypotheses[hid].confidence_log
assert entries[0]["rank"] == 1 and entries[0]["log_lr"] == pytest.approx(1.0)
assert entries[1]["rank"] == 2 and entries[1]["log_lr"] == pytest.approx(0.5)
@pytest.mark.asyncio
async def test_prior_prob_shifts_starting_log_odds(self, graph):
# prior 0.9 → log_odds ≈ +0.954