BattleTag/JANUS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Mixed_CFM: absorb Unified_CFM primitives; remove Unified_CFM

Browse Source 
Mixed_CFM was loading AdaLNBlock / SinusoidalTimeEmb / _sinkhorn_coupling
and flow-feature helpers from Unified_CFM via importlib spec hacks. Pulled
those symbols into Mixed_CFM/_layers.py (model primitives) and inlined
the flow-feature loader helpers into Mixed_CFM/data.py, then deleted
Unified_CFM/ entirely along with three dead aggregate shell scripts whose
referenced eval entry point (artifacts/verify_2026_04_24/) was already gone.

Verified: historic janus_iscxtor2016_seed42 checkpoint re-evaluated under
the absorbed code reproduces all 10 phase1 AUROC scores to 6 decimals;
same-seed retrain converges to within +/-0.001 on terminal_norm (residual
drift is CUDA non-determinism in MultiheadAttention + Sinkhorn argmax,
not the absorption).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
BattleTag
2026-05-11 14:17:04 +08:00
parent ee232058b1
commit ff0efa97bf
32 changed files with 175 additions and 2372 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
Mixed_CFM/data.py
View File

@@ -7,19 +7,116 @@ import pandas as pd
import sys as _sys
from pathlib import Path as _Path
_sys.path.insert(0, str(_Path(__file__).resolve().parents[1]))
from common.data_contract import PACKET_FEATURE_NAMES, PACKET_CONTINUOUS_CHANNEL_IDX, PACKET_BINARY_CHANNEL_IDX, fit_packet_stats as _fit_packet_stats, zscore as _zscore
import importlib.util as _ilu
_UDATA_NAME = 'unified_cfm_data'
if _UDATA_NAME not in _sys.modules:
_udata_spec = _ilu.spec_from_file_location(_UDATA_NAME, _Path(__file__).resolve().parents[1] / 'Unified_CFM' / 'data.py')
_udata = _ilu.module_from_spec(_udata_spec)
_sys.modules[_UDATA_NAME] = _udata
_udata_spec.loader.exec_module(_udata)
else:
_udata = _sys.modules[_UDATA_NAME]
DEFAULT_FLOW_META_COLUMNS = _udata.DEFAULT_FLOW_META_COLUMNS
_read_aligned_flow_features = _udata._read_aligned_flow_features
_preprocess_flow = _udata._preprocess_flow
from common.data_contract import (
PACKET_FEATURE_NAMES,
PACKET_CONTINUOUS_CHANNEL_IDX,
PACKET_BINARY_CHANNEL_IDX,
canonical_5tuple as _canonical_key,
fit_packet_stats as _fit_packet_stats,
zscore as _zscore,
)
DEFAULT_FLOW_META_COLUMNS = {'flow_id', 'label', 'day', 'service', 'src_ip', 'dst_ip', 'src_port', 'dst_port', 'protocol', 'timestamp', 'start_ts', 'n_pkts'}
def _read_flow_features(path: Path, *, expected_rows: int, feature_columns: Optional[list[str]]=None) -> tuple[np.ndarray, tuple[str, ...], np.ndarray | None]:
path = Path(path)
if path.suffix == '.npz':
data = np.load(path, allow_pickle=True)
x = data['features'].astype(np.float32)
raw_names = data['feature_names'] if 'feature_names' in data.files else np.arange(x.shape[1])
names = tuple((str(v) for v in raw_names))
flow_id = data['flow_id'] if 'flow_id' in data.files else None
elif path.suffix in ('.parquet', '.pq'):
df = pd.read_parquet(path)
flow_id = df['flow_id'].to_numpy() if 'flow_id' in df.columns else None
if feature_columns:
cols = feature_columns
else:
cols = [c for c in df.columns if c not in DEFAULT_FLOW_META_COLUMNS and pd.api.types.is_numeric_dtype(df[c])]
if not cols:
raise ValueError(f'no numeric flow feature columns found in {path}')
x = df[cols].to_numpy(dtype=np.float32)
names = tuple(cols)
else:
raise ValueError(f'unsupported flow feature file: {path}')
if len(x) != expected_rows:
raise ValueError(f'flow feature row count {len(x):,} != packet row count {expected_rows:,}')
x = np.nan_to_num(x, nan=0.0, posinf=0.0, neginf=0.0).astype(np.float32)
return (x, names, flow_id)
def _feature_columns_from_df(df: pd.DataFrame, requested: Optional[list[str]]) -> list[str]:
if requested:
return requested
return [c for c in df.columns if c not in DEFAULT_FLOW_META_COLUMNS and pd.api.types.is_numeric_dtype(df[c])]
def _align_flow_features_by_scan(feature_df: pd.DataFrame, packet_flows: pd.DataFrame, *, feature_columns: list[str]) -> tuple[np.ndarray, tuple[str, ...]]:
required = ['label', 'src_ip', 'src_port', 'dst_ip', 'dst_port', 'protocol']
missing_feature = [c for c in required if c not in feature_df.columns]
missing_packet = [c for c in required if c not in packet_flows.columns]
if missing_feature or missing_packet:
raise ValueError(f'scan alignment requires label + 5-tuple metadata. missing in feature_df={missing_feature}, packet_flows={missing_packet}')
packet_keys = [(str(lbl), _canonical_key(src, sp, dst, dp, proto)) for (lbl, src, sp, dst, dp, proto) in zip(packet_flows['label'].to_numpy(), packet_flows['src_ip'].to_numpy(), packet_flows['src_port'].to_numpy(), packet_flows['dst_ip'].to_numpy(), packet_flows['dst_port'].to_numpy(), packet_flows['protocol'].to_numpy())]
labels = feature_df['label'].to_numpy()
src_ip = feature_df['src_ip'].to_numpy()
src_port = feature_df['src_port'].to_numpy()
dst_ip = feature_df['dst_ip'].to_numpy()
dst_port = feature_df['dst_port'].to_numpy()
protocol = feature_df['protocol'].to_numpy()
matched: list[int] = []
j = 0
n_csv = len(feature_df)
for (i, target) in enumerate(packet_keys):
while j < n_csv:
cand = (str(labels[j]), _canonical_key(src_ip[j], src_port[j], dst_ip[j], dst_port[j], protocol[j]))
j += 1
if cand == target:
matched.append(j - 1)
break
else:
raise ValueError(f'failed to align packet flow row {i:,}/{len(packet_keys):,}; the CSV cache may not be the same one used for packet extraction')
print(f'[data] scan-aligned CSV flow features: matched={len(matched):,} from csv_rows={n_csv:,} skipped={matched[-1] + 1 - len(matched):,}')
x = feature_df.iloc[matched][feature_columns].to_numpy(dtype=np.float32)
x = np.nan_to_num(x, nan=0.0, posinf=0.0, neginf=0.0).astype(np.float32)
return (x, tuple(feature_columns))
def _read_aligned_flow_features(path: Path, packet_flows: pd.DataFrame, *, feature_columns: Optional[list[str]]=None, align: str='auto') -> tuple[np.ndarray, tuple[str, ...]]:
path = Path(path)
if align not in ('auto', 'row', 'scan'):
raise ValueError("flow_features_align must be 'auto', 'row', or 'scan'")
if path.suffix == '.npz':
(x, names, flow_id) = _read_flow_features(path, expected_rows=len(packet_flows), feature_columns=feature_columns)
packet_id = packet_flows['flow_id'].to_numpy() if 'flow_id' in packet_flows else None
if flow_id is not None and packet_id is not None and (not np.array_equal(flow_id, packet_id)):
raise ValueError('NPZ flow_id does not align with Packet_CFM flows')
return (x, names)
if path.suffix not in ('.parquet', '.pq'):
raise ValueError(f'unsupported flow feature file: {path}')
feature_df = pd.read_parquet(path)
cols = _feature_columns_from_df(feature_df, feature_columns)
if not cols:
raise ValueError(f'no numeric flow feature columns found in {path}')
packet_id = packet_flows['flow_id'].to_numpy() if 'flow_id' in packet_flows else None
if len(feature_df) == len(packet_flows):
feature_id = feature_df['flow_id'].to_numpy() if 'flow_id' in feature_df.columns else None
if feature_id is None or packet_id is None or np.array_equal(feature_id, packet_id):
x = feature_df[cols].to_numpy(dtype=np.float32)
x = np.nan_to_num(x, nan=0.0, posinf=0.0, neginf=0.0).astype(np.float32)
return (x, tuple(cols))
if align == 'row':
raise ValueError("flow_id mismatch with flow_features_align='row'")
if align == 'row':
raise ValueError(f'row alignment requested but feature rows={len(feature_df):,} packet rows={len(packet_flows):,}')
return _align_flow_features_by_scan(feature_df, packet_flows, feature_columns=cols)
def _preprocess_flow(train: np.ndarray, val: np.ndarray, attack: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
mean = train.mean(axis=0).astype(np.float32)
std = train.std(axis=0).astype(np.float32)
return (_zscore(train, mean, std), _zscore(val, mean, std), _zscore(attack, mean, std), mean, std)
@dataclass
class MixedData: