From 1fc0260309e3264c3ba23ffa0c13e0f456e39eb0 Mon Sep 17 00:00:00 2001 From: BattleTag Date: Thu, 7 May 2026 21:35:33 +0800 Subject: [PATCH] =?UTF-8?q?Rename=20A+C=20combo=20=E2=86=92=20JANUS:=20con?= =?UTF-8?q?figs,=20model=20dirs,=20scripts,=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...combo_seed42.yaml => cicddos2019_seed42.yaml} | 2 +- ...combo_seed43.yaml => cicddos2019_seed43.yaml} | 2 +- ...combo_seed44.yaml => cicddos2019_seed44.yaml} | 2 +- ..._combo_seed42.yaml => cicids2017_seed42.yaml} | 2 +- ..._combo_seed43.yaml => cicids2017_seed43.yaml} | 2 +- ..._combo_seed44.yaml => cicids2017_seed44.yaml} | 2 +- ...eed42.yaml => ciciot2023_route_c_seed42.yaml} | 4 +--- ...eed43.yaml => ciciot2023_route_c_seed43.yaml} | 4 +--- ...eed44.yaml => ciciot2023_route_c_seed44.yaml} | 4 +--- Mixed_CFM/configs/ciciot2023_seed42.yaml | 4 +++- Mixed_CFM/configs/ciciot2023_seed43.yaml | 4 +++- Mixed_CFM/configs/ciciot2023_seed44.yaml | 4 +++- ...combo_seed42.yaml => iscxtor2016_seed42.yaml} | 2 +- ...combo_seed43.yaml => iscxtor2016_seed43.yaml} | 2 +- ...combo_seed44.yaml => iscxtor2016_seed44.yaml} | 2 +- README.md | 4 ++-- RESULTS.md | 4 ++-- scripts/aggregate/aggregate_cross.py | 4 ++-- scripts/aggregate/aggregate_cross_matrix.py | 16 ++++++++-------- scripts/aggregate/aggregate_full_sota.py | 10 +++++----- scripts/aggregate/aggregate_score_router.py | 14 +++++++------- scripts/aggregate/aggregate_v2.py | 2 +- scripts/aggregate/run_full_cross_matrix.sh | 8 ++++---- ...{run_ac_combo_evals.sh => run_janus_evals.sh} | 14 +++++++------- 24 files changed, 59 insertions(+), 59 deletions(-) rename Mixed_CFM/configs/{cicddos2019_ac_combo_seed42.yaml => cicddos2019_seed42.yaml} (89%) rename Mixed_CFM/configs/{cicddos2019_ac_combo_seed43.yaml => cicddos2019_seed43.yaml} (89%) rename Mixed_CFM/configs/{cicddos2019_ac_combo_seed44.yaml => cicddos2019_seed44.yaml} (89%) rename Mixed_CFM/configs/{cicids2017_ac_combo_seed42.yaml => cicids2017_seed42.yaml} (88%) rename Mixed_CFM/configs/{cicids2017_ac_combo_seed43.yaml => cicids2017_seed43.yaml} (88%) rename Mixed_CFM/configs/{cicids2017_ac_combo_seed44.yaml => cicids2017_seed44.yaml} (88%) rename Mixed_CFM/configs/{ciciot2023_ac_combo_seed42.yaml => ciciot2023_route_c_seed42.yaml} (85%) rename Mixed_CFM/configs/{ciciot2023_ac_combo_seed43.yaml => ciciot2023_route_c_seed43.yaml} (85%) rename Mixed_CFM/configs/{ciciot2023_ac_combo_seed44.yaml => ciciot2023_route_c_seed44.yaml} (85%) rename Mixed_CFM/configs/{iscxtor2016_ac_combo_seed42.yaml => iscxtor2016_seed42.yaml} (88%) rename Mixed_CFM/configs/{iscxtor2016_ac_combo_seed43.yaml => iscxtor2016_seed43.yaml} (88%) rename Mixed_CFM/configs/{iscxtor2016_ac_combo_seed44.yaml => iscxtor2016_seed44.yaml} (88%) rename scripts/aggregate/{run_ac_combo_evals.sh => run_janus_evals.sh} (81%) diff --git a/Mixed_CFM/configs/cicddos2019_ac_combo_seed42.yaml b/Mixed_CFM/configs/cicddos2019_seed42.yaml similarity index 89% rename from Mixed_CFM/configs/cicddos2019_ac_combo_seed42.yaml rename to Mixed_CFM/configs/cicddos2019_seed42.yaml index fac97e9..a080008 100644 --- a/Mixed_CFM/configs/cicddos2019_ac_combo_seed42.yaml +++ b/Mixed_CFM/configs/cicddos2019_seed42.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_cicddos2019_seed42 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_cicddos2019_seed42 source_store: /home/chy/JANUS/datasets/cicddos2019/processed/full_store flows_parquet: /home/chy/JANUS/datasets/cicddos2019/processed/flows.parquet diff --git a/Mixed_CFM/configs/cicddos2019_ac_combo_seed43.yaml b/Mixed_CFM/configs/cicddos2019_seed43.yaml similarity index 89% rename from Mixed_CFM/configs/cicddos2019_ac_combo_seed43.yaml rename to Mixed_CFM/configs/cicddos2019_seed43.yaml index b8d1707..4313bd8 100644 --- a/Mixed_CFM/configs/cicddos2019_ac_combo_seed43.yaml +++ b/Mixed_CFM/configs/cicddos2019_seed43.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_cicddos2019_seed43 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_cicddos2019_seed43 source_store: /home/chy/JANUS/datasets/cicddos2019/processed/full_store flows_parquet: /home/chy/JANUS/datasets/cicddos2019/processed/flows.parquet diff --git a/Mixed_CFM/configs/cicddos2019_ac_combo_seed44.yaml b/Mixed_CFM/configs/cicddos2019_seed44.yaml similarity index 89% rename from Mixed_CFM/configs/cicddos2019_ac_combo_seed44.yaml rename to Mixed_CFM/configs/cicddos2019_seed44.yaml index 204506a..98c222b 100644 --- a/Mixed_CFM/configs/cicddos2019_ac_combo_seed44.yaml +++ b/Mixed_CFM/configs/cicddos2019_seed44.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_cicddos2019_seed44 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_cicddos2019_seed44 source_store: /home/chy/JANUS/datasets/cicddos2019/processed/full_store flows_parquet: /home/chy/JANUS/datasets/cicddos2019/processed/flows.parquet diff --git a/Mixed_CFM/configs/cicids2017_ac_combo_seed42.yaml b/Mixed_CFM/configs/cicids2017_seed42.yaml similarity index 88% rename from Mixed_CFM/configs/cicids2017_ac_combo_seed42.yaml rename to Mixed_CFM/configs/cicids2017_seed42.yaml index 899c8bd..393c290 100644 --- a/Mixed_CFM/configs/cicids2017_ac_combo_seed42.yaml +++ b/Mixed_CFM/configs/cicids2017_seed42.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_cicids2017_seed42 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_cicids2017_seed42 packets_npz: /home/chy/JANUS/datasets/cicids2017/processed/packets.npz flows_parquet: /home/chy/JANUS/datasets/cicids2017/processed/flows.parquet diff --git a/Mixed_CFM/configs/cicids2017_ac_combo_seed43.yaml b/Mixed_CFM/configs/cicids2017_seed43.yaml similarity index 88% rename from Mixed_CFM/configs/cicids2017_ac_combo_seed43.yaml rename to Mixed_CFM/configs/cicids2017_seed43.yaml index 9ef38cb..908527c 100644 --- a/Mixed_CFM/configs/cicids2017_ac_combo_seed43.yaml +++ b/Mixed_CFM/configs/cicids2017_seed43.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_cicids2017_seed43 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_cicids2017_seed43 packets_npz: /home/chy/JANUS/datasets/cicids2017/processed/packets.npz flows_parquet: /home/chy/JANUS/datasets/cicids2017/processed/flows.parquet diff --git a/Mixed_CFM/configs/cicids2017_ac_combo_seed44.yaml b/Mixed_CFM/configs/cicids2017_seed44.yaml similarity index 88% rename from Mixed_CFM/configs/cicids2017_ac_combo_seed44.yaml rename to Mixed_CFM/configs/cicids2017_seed44.yaml index 5f8dba8..34bd4b9 100644 --- a/Mixed_CFM/configs/cicids2017_ac_combo_seed44.yaml +++ b/Mixed_CFM/configs/cicids2017_seed44.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_cicids2017_seed44 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_cicids2017_seed44 packets_npz: /home/chy/JANUS/datasets/cicids2017/processed/packets.npz flows_parquet: /home/chy/JANUS/datasets/cicids2017/processed/flows.parquet diff --git a/Mixed_CFM/configs/ciciot2023_ac_combo_seed42.yaml b/Mixed_CFM/configs/ciciot2023_route_c_seed42.yaml similarity index 85% rename from Mixed_CFM/configs/ciciot2023_ac_combo_seed42.yaml rename to Mixed_CFM/configs/ciciot2023_route_c_seed42.yaml index 4ba011f..d5d9976 100644 --- a/Mixed_CFM/configs/ciciot2023_ac_combo_seed42.yaml +++ b/Mixed_CFM/configs/ciciot2023_route_c_seed42.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_ciciot2023_seed42 +save_dir: /home/chy/JANUS/artifacts/route_comparison/route_c_mixed_ciciot2023_seed42 source_store: /home/chy/JANUS/datasets/ciciot2023/processed/full_store flows_parquet: /home/chy/JANUS/datasets/ciciot2023/processed/full_store/flows.parquet @@ -40,5 +40,3 @@ use_ot: true lambda_disc: 1.0 device: auto - -reference_mode: causal_packets diff --git a/Mixed_CFM/configs/ciciot2023_ac_combo_seed43.yaml b/Mixed_CFM/configs/ciciot2023_route_c_seed43.yaml similarity index 85% rename from Mixed_CFM/configs/ciciot2023_ac_combo_seed43.yaml rename to Mixed_CFM/configs/ciciot2023_route_c_seed43.yaml index ad57179..72cac8b 100644 --- a/Mixed_CFM/configs/ciciot2023_ac_combo_seed43.yaml +++ b/Mixed_CFM/configs/ciciot2023_route_c_seed43.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_ciciot2023_seed43 +save_dir: /home/chy/JANUS/artifacts/route_comparison/route_c_mixed_ciciot2023_seed43 source_store: /home/chy/JANUS/datasets/ciciot2023/processed/full_store flows_parquet: /home/chy/JANUS/datasets/ciciot2023/processed/full_store/flows.parquet @@ -40,5 +40,3 @@ use_ot: true lambda_disc: 1.0 device: auto - -reference_mode: causal_packets diff --git a/Mixed_CFM/configs/ciciot2023_ac_combo_seed44.yaml b/Mixed_CFM/configs/ciciot2023_route_c_seed44.yaml similarity index 85% rename from Mixed_CFM/configs/ciciot2023_ac_combo_seed44.yaml rename to Mixed_CFM/configs/ciciot2023_route_c_seed44.yaml index 648be07..db93328 100644 --- a/Mixed_CFM/configs/ciciot2023_ac_combo_seed44.yaml +++ b/Mixed_CFM/configs/ciciot2023_route_c_seed44.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_ciciot2023_seed44 +save_dir: /home/chy/JANUS/artifacts/route_comparison/route_c_mixed_ciciot2023_seed44 source_store: /home/chy/JANUS/datasets/ciciot2023/processed/full_store flows_parquet: /home/chy/JANUS/datasets/ciciot2023/processed/full_store/flows.parquet @@ -40,5 +40,3 @@ use_ot: true lambda_disc: 1.0 device: auto - -reference_mode: causal_packets diff --git a/Mixed_CFM/configs/ciciot2023_seed42.yaml b/Mixed_CFM/configs/ciciot2023_seed42.yaml index d5d9976..75b8c55 100644 --- a/Mixed_CFM/configs/ciciot2023_seed42.yaml +++ b/Mixed_CFM/configs/ciciot2023_seed42.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_c_mixed_ciciot2023_seed42 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_ciciot2023_seed42 source_store: /home/chy/JANUS/datasets/ciciot2023/processed/full_store flows_parquet: /home/chy/JANUS/datasets/ciciot2023/processed/full_store/flows.parquet @@ -40,3 +40,5 @@ use_ot: true lambda_disc: 1.0 device: auto + +reference_mode: causal_packets diff --git a/Mixed_CFM/configs/ciciot2023_seed43.yaml b/Mixed_CFM/configs/ciciot2023_seed43.yaml index 72cac8b..d60a315 100644 --- a/Mixed_CFM/configs/ciciot2023_seed43.yaml +++ b/Mixed_CFM/configs/ciciot2023_seed43.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_c_mixed_ciciot2023_seed43 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_ciciot2023_seed43 source_store: /home/chy/JANUS/datasets/ciciot2023/processed/full_store flows_parquet: /home/chy/JANUS/datasets/ciciot2023/processed/full_store/flows.parquet @@ -40,3 +40,5 @@ use_ot: true lambda_disc: 1.0 device: auto + +reference_mode: causal_packets diff --git a/Mixed_CFM/configs/ciciot2023_seed44.yaml b/Mixed_CFM/configs/ciciot2023_seed44.yaml index db93328..87ed47d 100644 --- a/Mixed_CFM/configs/ciciot2023_seed44.yaml +++ b/Mixed_CFM/configs/ciciot2023_seed44.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_c_mixed_ciciot2023_seed44 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_ciciot2023_seed44 source_store: /home/chy/JANUS/datasets/ciciot2023/processed/full_store flows_parquet: /home/chy/JANUS/datasets/ciciot2023/processed/full_store/flows.parquet @@ -40,3 +40,5 @@ use_ot: true lambda_disc: 1.0 device: auto + +reference_mode: causal_packets diff --git a/Mixed_CFM/configs/iscxtor2016_ac_combo_seed42.yaml b/Mixed_CFM/configs/iscxtor2016_seed42.yaml similarity index 88% rename from Mixed_CFM/configs/iscxtor2016_ac_combo_seed42.yaml rename to Mixed_CFM/configs/iscxtor2016_seed42.yaml index 542378b..312bfef 100644 --- a/Mixed_CFM/configs/iscxtor2016_ac_combo_seed42.yaml +++ b/Mixed_CFM/configs/iscxtor2016_seed42.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_iscxtor2016_seed42 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_iscxtor2016_seed42 packets_npz: /home/chy/JANUS/datasets/iscxtor2016/processed/packets.npz flows_parquet: /home/chy/JANUS/datasets/iscxtor2016/processed/flows.parquet diff --git a/Mixed_CFM/configs/iscxtor2016_ac_combo_seed43.yaml b/Mixed_CFM/configs/iscxtor2016_seed43.yaml similarity index 88% rename from Mixed_CFM/configs/iscxtor2016_ac_combo_seed43.yaml rename to Mixed_CFM/configs/iscxtor2016_seed43.yaml index c243684..4d991b9 100644 --- a/Mixed_CFM/configs/iscxtor2016_ac_combo_seed43.yaml +++ b/Mixed_CFM/configs/iscxtor2016_seed43.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_iscxtor2016_seed43 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_iscxtor2016_seed43 packets_npz: /home/chy/JANUS/datasets/iscxtor2016/processed/packets.npz flows_parquet: /home/chy/JANUS/datasets/iscxtor2016/processed/flows.parquet diff --git a/Mixed_CFM/configs/iscxtor2016_ac_combo_seed44.yaml b/Mixed_CFM/configs/iscxtor2016_seed44.yaml similarity index 88% rename from Mixed_CFM/configs/iscxtor2016_ac_combo_seed44.yaml rename to Mixed_CFM/configs/iscxtor2016_seed44.yaml index 6b83be8..a8c19e7 100644 --- a/Mixed_CFM/configs/iscxtor2016_ac_combo_seed44.yaml +++ b/Mixed_CFM/configs/iscxtor2016_seed44.yaml @@ -1,5 +1,5 @@ -save_dir: /home/chy/JANUS/artifacts/route_comparison/route_ac_combo_iscxtor2016_seed44 +save_dir: /home/chy/JANUS/artifacts/route_comparison/janus_iscxtor2016_seed44 packets_npz: /home/chy/JANUS/datasets/iscxtor2016/processed/packets.npz flows_parquet: /home/chy/JANUS/datasets/iscxtor2016/processed/flows.parquet diff --git a/README.md b/README.md index 1363632..728bfe7 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ The 9-d packet schema and 20-d flow schema are FIXED in `common/data_contract.py ```bash # Train JANUS on CICIDS2017 (3 seeds available: 42, 43, 44) cd Mixed_CFM -uv run --no-sync python train.py --config configs/cicids2017_ac_combo_seed42.yaml +uv run --no-sync python train.py --config configs/cicids2017_seed42.yaml # Phase-1 evaluation: per-attack-class AUROC + 10-d score export uv run --no-sync python eval_phase1.py \ @@ -92,7 +92,7 @@ uv run --no-sync python eval_cross.py \ --out ``` -JANUS hyper-parameters (locked in `Mixed_CFM/configs/_ac_combo_seed*.yaml`): +JANUS hyper-parameters (locked in `Mixed_CFM/configs/_seed*.yaml`): ```yaml T: 64 # max packet sequence length diff --git a/RESULTS.md b/RESULTS.md index 1122725..6a69551 100644 --- a/RESULTS.md +++ b/RESULTS.md @@ -25,7 +25,7 @@ All numbers reported are 3-seed mean ± std. Two model families are tracked: - **Unified_CFM** (legacy / our previous internal recipe): single Transformer over [FLOW + packets] with Phase-2 consistency loss; λ=0.3. Strongest single-fixed-score (`terminal_norm`) within-dataset baseline. -- **JANUS = A+C combo** (current main line, 2026-05-01): see above. +- **JANUS** (current main line, 2026-05-01): see above. **New SOTA on cross-dataset transfer** under Mahalanobis auto-routing; matches legacy within-dataset under the same protocol. See `artifacts/route_comparison/SCORE_ROUTER.md`. @@ -300,7 +300,7 @@ scores are diagnostic only. ```yaml # CURRENT SOTA: JANUS (Mixed_CFM + causal-packet attention). -# Configs at: Mixed_CFM/configs/_ac_combo_seed{42,43,44}.yaml +# Configs at: Mixed_CFM/configs/_seed{42,43,44}.yaml model: T: 64 d_model: 128 diff --git a/scripts/aggregate/aggregate_cross.py b/scripts/aggregate/aggregate_cross.py index 839118d..aa2f63d 100644 --- a/scripts/aggregate/aggregate_cross.py +++ b/scripts/aggregate/aggregate_cross.py @@ -7,7 +7,7 @@ from sklearn.metrics import roc_auc_score ROOT = Path(__file__).resolve().parents[2] / 'artifacts' / 'route_comparison' CROSS_DIR = ROOT / 'cross' NAME_RE = re.compile('^(?P.+?)_seed(?P\\d+)_to_(?Pcicids2017|cicddos2019)$') -ROUTES = [('baseline', 'baseline'), ('A: causal', 'route_a_causal'), ('B: spectral', 'route_b_spectral'), ('C: mixed', 'route_c_mixed'), ('A+C combo', 'route_ac_combo')] +ROUTES = [('baseline', 'baseline'), ('A: causal', 'route_a_causal'), ('B: spectral', 'route_b_spectral'), ('C: mixed', 'route_c_mixed'), ('JANUS', 'route_janus')] TARGETS = ['cicids2017', 'cicddos2019'] PRIMARY_SCORES = ['terminal_norm', 'terminal_flow', 'terminal_packet', 'flow_consistency', 'packet_consistency', 'consistency_total', 'causal_surprisal_packet_median', 'causal_surprisal_total', 'direction_drift_packet_median', 'pna_packet_median', 'kappa2_speed2norm_packet_median', 'curvature_packet', 'disc_nll_total', 'disc_nll_ch3', 'disc_nll_ch7'] @@ -67,7 +67,7 @@ def main() -> None: header = '| Route | within-CICIoT2023 (ref) | → CICIDS2017 | → CICDDoS2019 |' rows.append(header) rows.append('|---|---|---|---|') - within_fallback = {'baseline': (0.9612, 0.0017), 'A: causal': (0.9636, 0.0006), 'B: spectral': (0.9619, 0.0013), 'C: mixed': (0.9625, 0.0028), 'A+C combo': (0.9587, 0.0017)} + within_fallback = {'baseline': (0.9612, 0.0017), 'A: causal': (0.9636, 0.0006), 'B: spectral': (0.9619, 0.0013), 'C: mixed': (0.9625, 0.0028), 'JANUS': (0.9587, 0.0017)} within_terminal: dict[str, tuple[float, float]] = {} for (label, prefix) in ROUTES: within_seeds = sorted(ROOT.glob(f'{prefix}_seed*/phase1_summary.json')) diff --git a/scripts/aggregate/aggregate_cross_matrix.py b/scripts/aggregate/aggregate_cross_matrix.py index ff73823..78adb83 100644 --- a/scripts/aggregate/aggregate_cross_matrix.py +++ b/scripts/aggregate/aggregate_cross_matrix.py @@ -41,11 +41,11 @@ def _mahal_eval(npz_path: Path, val_prefix: str, atk_prefix: str) -> float: return float('nan') def _within_mahal(ds: str, seed: int) -> float: - md = ROOT / f'route_ac_combo_{ds}_seed{seed}' + md = ROOT / f'janus_{ds}_seed{seed}' return _mahal_eval(md / 'phase1_scores.npz', 'val_', 'atk_') def _within_terminal_norm(ds: str, seed: int) -> float: - f = ROOT / f'route_ac_combo_{ds}_seed{seed}' / 'phase1_summary.json' + f = ROOT / f'janus_{ds}_seed{seed}' / 'phase1_summary.json' if not f.exists(): return float('nan') return json.loads(f.read_text())['overall'].get('terminal_norm', {}).get('auroc', float('nan')) @@ -59,9 +59,9 @@ def _src_aliases(src: str) -> list[str]: def _cross_mahal(src: str, tgt: str, seed: int) -> float: candidates = [] for alias in _src_aliases(src): - candidates.append(CROSS_DIR / f'route_ac_combo_seed{seed}_{alias}_to_{tgt}.npz') + candidates.append(CROSS_DIR / f'janus_seed{seed}_{alias}_to_{tgt}.npz') if src == 'ciciot2023': - candidates.append(CROSS_DIR / f'route_ac_combo_seed{seed}_to_{tgt}.npz') + candidates.append(CROSS_DIR / f'janus_seed{seed}_to_{tgt}.npz') for c in candidates: if c.exists(): return _mahal_eval(c, 'b_', 'a_') @@ -70,9 +70,9 @@ def _cross_mahal(src: str, tgt: str, seed: int) -> float: def _cross_terminal_norm(src: str, tgt: str, seed: int) -> float: candidates = [] for alias in _src_aliases(src): - candidates.append(CROSS_DIR / f'route_ac_combo_seed{seed}_{alias}_to_{tgt}.json') + candidates.append(CROSS_DIR / f'janus_seed{seed}_{alias}_to_{tgt}.json') if src == 'ciciot2023': - candidates.append(CROSS_DIR / f'route_ac_combo_seed{seed}_to_{tgt}.json') + candidates.append(CROSS_DIR / f'janus_seed{seed}_to_{tgt}.json') for c in candidates: if c.exists(): d = json.loads(c.read_text()) @@ -89,10 +89,10 @@ def _ms(vals: list[float]) -> str: def main() -> None: rows: list[str] = [] - rows.append('# Full 4×4 Cross Matrix — A+C combo + Mahalanobis-OAS') + rows.append('# Full 4×4 Cross Matrix — JANUS + Mahalanobis-OAS') rows.append('') rows.append('3-seed mean ± std. Diagonal = within-dataset; off-diagonal = cross.') - rows.append('Aggregator: Mahalanobis-OAS over 10-d A+C combo score vector,') + rows.append('Aggregator: Mahalanobis-OAS over 10-d JANUS score vector,') rows.append('fit on **target-dataset benign val only** (no attack labels).') rows.append('') rows.append('## Mahalanobis-OAS AUROC (4×4)') diff --git a/scripts/aggregate/aggregate_full_sota.py b/scripts/aggregate/aggregate_full_sota.py index c09baf3..7e69914 100644 --- a/scripts/aggregate/aggregate_full_sota.py +++ b/scripts/aggregate/aggregate_full_sota.py @@ -5,7 +5,7 @@ from pathlib import Path import numpy as np ROOT = Path(__file__).resolve().parents[2] / 'artifacts' / 'route_comparison' SEED_RE = re.compile('_seed(\\d+)$') -EXISTING_SOTA = {'ISCXTor2016 (NonTor → Tor)': {'shafir_baseline': 0.8731, 'shafir_ref': 'Table VI', 'ours_existing': (0.9945, 0.0011), 'ours_score': 'terminal_norm', 'sigma': 0.1, 'ac_prefix': 'route_ac_combo_iscxtor2016'}, 'CICIDS2017 within (Shafir 10k/10k)': {'shafir_baseline': 0.9303, 'shafir_ref': 'Table VII', 'ours_existing': (0.9858, 0.0021), 'ours_score': 'terminal_norm', 'sigma': 0.6, 'ac_prefix': 'route_ac_combo_cicids2017'}, 'CICDDoS2019 within': {'shafir_baseline': 0.93, 'shafir_ref': 'Table IX, row 1', 'ours_existing': (0.996, 0.001), 'ours_score': 'terminal_norm', 'sigma': 0.1, 'ac_prefix': 'route_ac_combo_cicddos2019'}, 'CICIoT2023 within (multi-seed)': {'shafir_baseline': None, 'shafir_ref': None, 'ours_existing': (0.9612, 0.0017), 'ours_score': 'terminal_norm', 'sigma': 0.1, 'ac_prefix': 'route_ac_combo_ciciot2023'}} +EXISTING_SOTA = {'ISCXTor2016 (NonTor → Tor)': {'shafir_baseline': 0.8731, 'shafir_ref': 'Table VI', 'ours_existing': (0.9945, 0.0011), 'ours_score': 'terminal_norm', 'sigma': 0.1, 'ac_prefix': 'janus_iscxtor2016'}, 'CICIDS2017 within (Shafir 10k/10k)': {'shafir_baseline': 0.9303, 'shafir_ref': 'Table VII', 'ours_existing': (0.9858, 0.0021), 'ours_score': 'terminal_norm', 'sigma': 0.6, 'ac_prefix': 'janus_cicids2017'}, 'CICDDoS2019 within': {'shafir_baseline': 0.93, 'shafir_ref': 'Table IX, row 1', 'ours_existing': (0.996, 0.001), 'ours_score': 'terminal_norm', 'sigma': 0.1, 'ac_prefix': 'janus_cicddos2019'}, 'CICIoT2023 within (multi-seed)': {'shafir_baseline': None, 'shafir_ref': None, 'ours_existing': (0.9612, 0.0017), 'ours_score': 'terminal_norm', 'sigma': 0.1, 'ac_prefix': 'janus_ciciot2023'}} def _seeds(prefix: str) -> dict[int, Path]: out = {} @@ -26,16 +26,16 @@ def _mean_std(vs: list[float]) -> tuple[float, float]: def main() -> None: rows: list[str] = [] - rows.append('# SOTA Comparison: A+C combo vs existing UnifiedCFM') + rows.append('# SOTA Comparison: JANUS vs existing UnifiedCFM') rows.append('') rows.append('All 4 datasets, 3 seeds each, within-dataset Shafir 10K/10K protocol.') rows.append('Existing UnifiedCFM uses Phase-2 consistency loss (λ_flow=λ_packet=0.3).') - rows.append('A+C combo uses Mixed_CFM (continuous CFM + DFM) + causal-packet attention,') + rows.append('JANUS uses Mixed_CFM (continuous CFM + DFM) + causal-packet attention,') rows.append('**no Phase-2 consistency loss**. lambda_disc=1.0, sigma=0.1, use_ot=True.') rows.append('') - rows.append("## Headline: A+C combo's best score per dataset") + rows.append("## Headline: JANUS's best score per dataset") rows.append('') - rows.append('| Dataset | Shafir 2026 | Existing UnifiedCFM (SOTA) | A+C combo `terminal_norm` | A+C combo `terminal_packet` | A+C combo `disc_nll_total` | A+C best | New SOTA? |') + rows.append('| Dataset | Shafir 2026 | Existing UnifiedCFM (SOTA) | JANUS `terminal_norm` | JANUS `terminal_packet` | JANUS `disc_nll_total` | JANUS best | New SOTA? |') rows.append('|---|---|---|---|---|---|---|---|') for (label, meta) in EXISTING_SOTA.items(): seeds = _seeds(meta['ac_prefix']) diff --git a/scripts/aggregate/aggregate_score_router.py b/scripts/aggregate/aggregate_score_router.py index ce843b9..975f12e 100644 --- a/scripts/aggregate/aggregate_score_router.py +++ b/scripts/aggregate/aggregate_score_router.py @@ -102,20 +102,20 @@ def main() -> None: rows.append('# Score-vector auto-selection: max-of-|z| / Mahalanobis vs fixed scores') rows.append('') rows.append('Aggregators are fit on **benign val only** (no attack labels). All numbers') - rows.append('are 3-seed mean ± std on A+C combo (Mixed_CFM + causal-packet attention).') + rows.append('are 3-seed mean ± std on JANUS (Mixed_CFM + causal-packet attention).') rows.append('') rows.append('Note on fairness: `auc_best_fixed` is selection-biased (picks per-dataset best') rows.append('score post-hoc on test set). `max_abs_z` and `mahalanobis` are NOT — they only') rows.append('use benign val to fit aggregator parameters.') rows.append('') - rows.append("## Within-dataset(A+C combo on each dataset's own benign/attack)") + rows.append("## Within-dataset(JANUS on each dataset's own benign/attack)") rows.append('') rows.append('| Dataset | term_norm | best fixed | max-\\|z\\| (all) | mahal-OAS (all) | **mahal-OAS (term3)** | **mahal-OAS (disc7)** |') rows.append('|---|---|---|---|---|---|---|') for ds in WITHIN_DATASETS: rows_per_seed: list[dict] = [] for s in SEEDS: - md = ROOT / f'route_ac_combo_{ds}_seed{s}' + md = ROOT / f'janus_{ds}_seed{s}' npz = md / 'phase1_scores.npz' if not npz.exists(): continue @@ -129,14 +129,14 @@ def main() -> None: return f'{m:.4f} ± {sd:.4f}' rows.append(f"| {ds} | {col('auc_term_norm')} | {col('auc_best_fixed')} | {col('auc_max_abs_z_all')} | {col('auc_mahal_oas_all')} | **{col('auc_mahal_oas_terminal3')}** | **{col('auc_mahal_oas_disc7')}** |") rows.append('') - rows.append('## Cross-dataset(A+C combo trained on CICIoT2023 → eval on target)') + rows.append('## Cross-dataset(JANUS trained on CICIoT2023 → eval on target)') rows.append('') rows.append('| Target | term_norm | best fixed | max-\\|z\\| (all) | mahal-OAS (all) | **mahal-OAS (term3)** | **mahal-OAS (disc7)** |') rows.append('|---|---|---|---|---|---|---|') for tgt in CROSS_TARGETS: rows_per_seed: list[dict] = [] for s in SEEDS: - npz = CROSS_DIR / f'route_ac_combo_seed{s}_to_{tgt}.npz' + npz = CROSS_DIR / f'janus_seed{s}_to_{tgt}.npz' if not npz.exists(): continue rows_per_seed.append(_evaluate(npz, 'b_', 'a_')) @@ -156,7 +156,7 @@ def main() -> None: for ds in WITHIN_DATASETS: cells = [f'within {ds}'] for s in SEEDS: - npz = ROOT / f'route_ac_combo_{ds}_seed{s}/phase1_scores.npz' + npz = ROOT / f'janus_{ds}_seed{s}/phase1_scores.npz' if not npz.exists(): cells.append('—') continue @@ -166,7 +166,7 @@ def main() -> None: for tgt in CROSS_TARGETS: cells = [f'cross→{tgt}'] for s in SEEDS: - npz = CROSS_DIR / f'route_ac_combo_seed{s}_to_{tgt}.npz' + npz = CROSS_DIR / f'janus_seed{s}_to_{tgt}.npz' if not npz.exists(): cells.append('—') continue diff --git a/scripts/aggregate/aggregate_v2.py b/scripts/aggregate/aggregate_v2.py index 648f0d1..02dabcf 100644 --- a/scripts/aggregate/aggregate_v2.py +++ b/scripts/aggregate/aggregate_v2.py @@ -6,7 +6,7 @@ import numpy as np from sklearn.metrics import roc_auc_score ROOT = Path(__file__).resolve().parents[2] / 'artifacts' / 'route_comparison' SEED_RE = re.compile('_seed(\\d+)$') -ROUTES = [('baseline', 'baseline_ciciot2023'), ('A: causal', 'route_a_causal_ciciot2023'), ('B: spectral', 'route_b_spectral_ciciot2023'), ('C: mixed', 'route_c_mixed_ciciot2023'), ('A+C combo', 'route_ac_combo_ciciot2023')] +ROUTES = [('baseline', 'baseline_ciciot2023'), ('A: causal', 'route_a_causal_ciciot2023'), ('B: spectral', 'route_b_spectral_ciciot2023'), ('C: mixed', 'route_c_mixed_ciciot2023'), ('JANUS', 'janus_ciciot2023')] def _seeds(prefix: str) -> dict[int, Path]: out = {} diff --git a/scripts/aggregate/run_full_cross_matrix.sh b/scripts/aggregate/run_full_cross_matrix.sh index fa83217..1ba75e9 100755 --- a/scripts/aggregate/run_full_cross_matrix.sh +++ b/scripts/aggregate/run_full_cross_matrix.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Run all missing cross-direction evals for A+C combo. +# Run all missing cross-direction evals for JANUS. # Targets are routed to packets-npz or full_store as appropriate. set -e @@ -32,8 +32,8 @@ TGT_ciciot2023_label=normal run_one() { local gpu=$1 src=$2 tgt=$3 seed=$4 - local md=${ROOT}/artifacts/route_comparison/route_ac_combo_${src}_seed${seed} - local out=${CROSS_DIR}/route_ac_combo_seed${seed}_${src}_to_${tgt}.json + local md=${ROOT}/artifacts/route_comparison/janus_${src}_seed${seed} + local out=${CROSS_DIR}/janus_seed${seed}_${src}_to_${tgt}.json if [ -f "${out}" ]; then echo "[skip] ${src}→${tgt} seed${seed}"; return; fi if [ ! -f "${md}/model.pt" ]; then echo "[missing] ${md}/model.pt"; return; fi @@ -56,7 +56,7 @@ run_one() { ${tgt_args} \ --out ${out} \ --n-benign 10000 --seed 42 --T 64 --batch-size 256 --n-steps 16 \ - > ${CROSS_DIR}/route_ac_combo_seed${seed}_${src}_to_${tgt}.log 2>&1 + > ${CROSS_DIR}/janus_seed${seed}_${src}_to_${tgt}.log 2>&1 } # 8 missing directions × 3 seeds = 24 evals diff --git a/scripts/aggregate/run_ac_combo_evals.sh b/scripts/aggregate/run_janus_evals.sh similarity index 81% rename from scripts/aggregate/run_ac_combo_evals.sh rename to scripts/aggregate/run_janus_evals.sh index f13c283..378a410 100755 --- a/scripts/aggregate/run_ac_combo_evals.sh +++ b/scripts/aggregate/run_janus_evals.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Phase1 + cross eval for the 3 A+C combo seeds. +# Phase1 + cross eval for the 3 JANUS seeds. set -e ROOT=/home/chy/JANUS MIXED_PHASE1=${ROOT}/Mixed_CFM/eval_phase1.py @@ -10,7 +10,7 @@ mkdir -p ${CROSS_DIR} # GPU 0: phase1 + cross→IDS2017 for all 3 seeds { for seed in 42 43 44; do - md=${ROOT}/artifacts/route_comparison/route_ac_combo_ciciot2023_seed${seed} + md=${ROOT}/artifacts/route_comparison/janus_ciciot2023_seed${seed} [ -f "${md}/model.pt" ] || { echo "[wait] seed${seed} model.pt not yet"; continue; } if [ ! -f "${md}/phase1_summary.json" ]; then @@ -23,7 +23,7 @@ for seed in 42 43 44; do > ${md}/phase1.log 2>&1 fi - ids_out=${CROSS_DIR}/route_ac_combo_seed${seed}_to_cicids2017.json + ids_out=${CROSS_DIR}/janus_seed${seed}_to_cicids2017.json if [ ! -f "${ids_out}" ]; then echo "[gpu0 cross→ids2017] seed${seed}" cd ${ROOT}/Mixed_CFM @@ -34,7 +34,7 @@ for seed in 42 43 44; do --target-flow-features ${ROOT}/datasets/cicids2017/processed/flow_features.parquet \ --out ${ids_out} \ --n-benign 10000 --n-attack 10000 --seed 42 --T 64 --batch-size 256 --n-steps 16 \ - > ${CROSS_DIR}/route_ac_combo_seed${seed}_to_cicids2017.log 2>&1 + > ${CROSS_DIR}/janus_seed${seed}_to_cicids2017.log 2>&1 fi done echo "[gpu0 done]" @@ -44,10 +44,10 @@ GPU0=$! # GPU 1: cross→DDoS19 for all 3 seeds { for seed in 42 43 44; do - md=${ROOT}/artifacts/route_comparison/route_ac_combo_ciciot2023_seed${seed} + md=${ROOT}/artifacts/route_comparison/janus_ciciot2023_seed${seed} [ -f "${md}/model.pt" ] || { echo "[wait] seed${seed} model.pt not yet"; continue; } - ddos_out=${CROSS_DIR}/route_ac_combo_seed${seed}_to_cicddos2019.json + ddos_out=${CROSS_DIR}/janus_seed${seed}_to_cicddos2019.json if [ ! -f "${ddos_out}" ]; then echo "[gpu1 cross→ddos19] seed${seed}" cd ${ROOT}/Mixed_CFM @@ -58,7 +58,7 @@ for seed in 42 43 44; do --target-flow-features ${ROOT}/datasets/cicddos2019/processed/flow_features.parquet \ --out ${ddos_out} \ --n-benign 10000 --n-attack 10000 --seed 42 --T 64 --batch-size 256 --n-steps 16 \ - > ${CROSS_DIR}/route_ac_combo_seed${seed}_to_cicddos2019.log 2>&1 + > ${CROSS_DIR}/janus_seed${seed}_to_cicddos2019.log 2>&1 fi done echo "[gpu1 done]"