Update README headline + 3x3 cross table; add cross_3x3 tooling to scripts/aggregate
This commit is contained in:
122
scripts/aggregate/cross_3x3_table.py
Normal file
122
scripts/aggregate/cross_3x3_table.py
Normal file
@@ -0,0 +1,122 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
from sklearn.covariance import OAS
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
ROOT = Path("/home/chy/JANUS/artifacts/route_comparison")
|
||||
CROSS = ROOT / "cross"
|
||||
DATASETS = ["cicids2017", "cicddos2019", "ciciot2023"]
|
||||
SEEDS = [42, 43, 44]
|
||||
|
||||
|
||||
def load_cell(src, tgt, seed):
|
||||
if src == tgt:
|
||||
path = ROOT / f"janus_{src}_seed{seed}/phase1_scores.npz"
|
||||
prefix_b, prefix_a = "val_", "atk_"
|
||||
else:
|
||||
path = CROSS / f"janus_seed{seed}_{src}_to_{tgt}.npz"
|
||||
prefix_b, prefix_a = "b_", "a_"
|
||||
z = np.load(path, allow_pickle=True)
|
||||
keys = sorted(
|
||||
k.replace(prefix_b, "")
|
||||
for k in z.files
|
||||
if k.startswith(prefix_b) and not k.endswith("labels")
|
||||
)
|
||||
val_S = np.stack([z[f"{prefix_b}{k}"] for k in keys], axis=1)
|
||||
atk_S = np.stack([z[f"{prefix_a}{k}"] for k in keys], axis=1)
|
||||
val_S = np.nan_to_num(val_S, nan=0.0, posinf=1e6, neginf=-1e6)
|
||||
atk_S = np.nan_to_num(atk_S, nan=0.0, posinf=1e6, neginf=-1e6)
|
||||
return val_S, atk_S
|
||||
|
||||
|
||||
def mahal_oas_auroc(val_S, atk_S):
|
||||
K = val_S.shape[1]
|
||||
mu = val_S.mean(axis=0)
|
||||
oas = OAS().fit(val_S)
|
||||
inv_cov = np.linalg.inv(oas.covariance_ + 1e-9 * np.eye(K))
|
||||
|
||||
def d2(S):
|
||||
d = S - mu
|
||||
return np.einsum("ni,ij,nj->n", d, inv_cov, d)
|
||||
|
||||
s_val = d2(val_S)
|
||||
s_atk = d2(atk_S)
|
||||
s = np.r_[s_val, s_atk]
|
||||
s = np.nan_to_num(s, nan=0.0, posinf=1e12, neginf=-1e12)
|
||||
y = np.r_[np.zeros(val_S.shape[0]), np.ones(atk_S.shape[0])]
|
||||
return float(roc_auc_score(y, s))
|
||||
|
||||
|
||||
cells = {}
|
||||
sample_counts = {}
|
||||
for src in DATASETS:
|
||||
for tgt in DATASETS:
|
||||
aucs = []
|
||||
n_val_seen = n_atk_seen = None
|
||||
for s in SEEDS:
|
||||
val_S, atk_S = load_cell(src, tgt, s)
|
||||
auc = mahal_oas_auroc(val_S, atk_S)
|
||||
aucs.append(auc)
|
||||
n_val_seen, n_atk_seen = val_S.shape[0], atk_S.shape[0]
|
||||
a = np.array(aucs)
|
||||
cells[(src, tgt)] = (a.mean(), a.std())
|
||||
sample_counts[(src, tgt)] = (n_val_seen, n_atk_seen)
|
||||
|
||||
|
||||
def short(name):
|
||||
return {"cicids2017": "CICIDS17", "cicddos2019": "CICDDoS19", "ciciot2023": "CICIoT23"}[name]
|
||||
|
||||
|
||||
print("# 3×3 cross-dataset AUROC matrix (Mahalanobis-OAS, 3-seed mean ± std)\n")
|
||||
print("Rows = source (training), columns = target (test). Diagonal = within-dataset.")
|
||||
print("Aggregator fit on target benign val only; tested on target benign + ALL target attacks.\n")
|
||||
|
||||
header = "| Source ↓ / Target → | " + " | ".join(short(t) for t in DATASETS) + " |"
|
||||
sep = "|" + "|".join(["---"] * (len(DATASETS) + 1)) + "|"
|
||||
print(header)
|
||||
print(sep)
|
||||
for src in DATASETS:
|
||||
row = [short(src)]
|
||||
for tgt in DATASETS:
|
||||
m, sd = cells[(src, tgt)]
|
||||
cell = f"{m:.4f} ± {sd:.4f}"
|
||||
if src == tgt:
|
||||
cell = f"_{cell}_"
|
||||
row.append(cell)
|
||||
print("| " + " | ".join(row) + " |")
|
||||
|
||||
print("\n## Sample counts (target benign / all target attacks)\n")
|
||||
print(header)
|
||||
print(sep)
|
||||
for src in DATASETS:
|
||||
row = [short(src)]
|
||||
for tgt in DATASETS:
|
||||
n_b, n_a = sample_counts[(src, tgt)]
|
||||
row.append(f"{n_b}b / {n_a}a")
|
||||
print("| " + " | ".join(row) + " |")
|
||||
|
||||
out_md = ROOT / "CROSS_MATRIX_3x3.md"
|
||||
with out_md.open("w") as f:
|
||||
f.write("# 3×3 cross-dataset AUROC matrix (Mahalanobis-OAS, 3-seed mean ± std)\n\n")
|
||||
f.write("Rows = source (training), columns = target (test). Diagonal italic = within-dataset.\n")
|
||||
f.write("Aggregator fit on target benign val only; tested on target benign + ALL target attacks.\n\n")
|
||||
f.write(header + "\n" + sep + "\n")
|
||||
for src in DATASETS:
|
||||
row = [short(src)]
|
||||
for tgt in DATASETS:
|
||||
m, sd = cells[(src, tgt)]
|
||||
cell = f"{m:.4f} ± {sd:.4f}"
|
||||
if src == tgt:
|
||||
cell = f"_{cell}_"
|
||||
row.append(cell)
|
||||
f.write("| " + " | ".join(row) + " |\n")
|
||||
f.write("\n## Sample counts (target benign / all target attacks)\n\n")
|
||||
f.write(header + "\n" + sep + "\n")
|
||||
for src in DATASETS:
|
||||
row = [short(src)]
|
||||
for tgt in DATASETS:
|
||||
n_b, n_a = sample_counts[(src, tgt)]
|
||||
row.append(f"{n_b}b / {n_a}a")
|
||||
f.write("| " + " | ".join(row) + " |\n")
|
||||
print(f"\n[wrote] {out_md}")
|
||||
62
scripts/aggregate/run_cross_3x3.sh
Executable file
62
scripts/aggregate/run_cross_3x3.sh
Executable file
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
ROOT=/home/chy/JANUS
|
||||
EVAL=${ROOT}/Mixed_CFM/eval_cross.py
|
||||
CROSS_DIR=${ROOT}/artifacts/route_comparison/cross
|
||||
mkdir -p ${CROSS_DIR}
|
||||
|
||||
declare -A STORE FLOWS FEATS
|
||||
STORE[cicids2017]=${ROOT}/datasets/cicids2017/processed/full_store
|
||||
FLOWS[cicids2017]=${ROOT}/datasets/cicids2017/processed/flows.parquet
|
||||
FEATS[cicids2017]=${ROOT}/datasets/cicids2017/processed/flow_features.parquet
|
||||
STORE[cicddos2019]=${ROOT}/datasets/cicddos2019/processed/full_store
|
||||
FLOWS[cicddos2019]=${ROOT}/datasets/cicddos2019/processed/flows.parquet
|
||||
FEATS[cicddos2019]=${ROOT}/datasets/cicddos2019/processed/flow_features.parquet
|
||||
STORE[ciciot2023]=${ROOT}/datasets/ciciot2023/processed/full_store
|
||||
FLOWS[ciciot2023]=${ROOT}/datasets/ciciot2023/processed/full_store/flows.parquet
|
||||
FEATS[ciciot2023]=${ROOT}/datasets/ciciot2023/processed/flow_features.parquet
|
||||
|
||||
run_one() {
|
||||
local gpu=$1 src=$2 tgt=$3 seed=$4
|
||||
local md=${ROOT}/artifacts/route_comparison/janus_${src}_seed${seed}
|
||||
local out=${CROSS_DIR}/janus_seed${seed}_${src}_to_${tgt}.json
|
||||
if [ -f "${out}" ]; then echo "[skip] ${src}→${tgt} seed${seed}"; return; fi
|
||||
if [ ! -f "${md}/model.pt" ]; then echo "[missing model] ${md}/model.pt"; return; fi
|
||||
echo "[gpu${gpu}] ${src} → ${tgt} seed${seed}"
|
||||
cd ${ROOT}/Mixed_CFM
|
||||
CUDA_VISIBLE_DEVICES=${gpu} stdbuf -oL uv run --no-sync python -u ${EVAL} \
|
||||
--model-dir ${md} \
|
||||
--target-store ${STORE[$tgt]} --target-flows ${FLOWS[$tgt]} --target-flow-features ${FEATS[$tgt]} \
|
||||
--benign-label normal --n-benign 10000 --n-attack 1000000 \
|
||||
--out ${out} --seed ${seed} --T 64 --batch-size 512 --n-steps 16 \
|
||||
> ${CROSS_DIR}/janus_seed${seed}_${src}_to_${tgt}.log 2>&1
|
||||
}
|
||||
|
||||
GPU0_DIRS=("cicids2017:cicddos2019" "cicids2017:ciciot2023" "cicddos2019:cicids2017")
|
||||
GPU1_DIRS=("cicddos2019:ciciot2023" "ciciot2023:cicids2017" "ciciot2023:cicddos2019")
|
||||
|
||||
{
|
||||
for dir in "${GPU0_DIRS[@]}"; do
|
||||
src=${dir%:*}; tgt=${dir#*:}
|
||||
for seed in 42 43 44; do
|
||||
run_one 0 ${src} ${tgt} ${seed}
|
||||
done
|
||||
done
|
||||
echo "[gpu0 done]"
|
||||
} &
|
||||
G0=$!
|
||||
|
||||
{
|
||||
for dir in "${GPU1_DIRS[@]}"; do
|
||||
src=${dir%:*}; tgt=${dir#*:}
|
||||
for seed in 42 43 44; do
|
||||
run_one 1 ${src} ${tgt} ${seed}
|
||||
done
|
||||
done
|
||||
echo "[gpu1 done]"
|
||||
} &
|
||||
G1=$!
|
||||
|
||||
wait $G0 $G1
|
||||
echo "[all done]"
|
||||
ls ${CROSS_DIR}/*.json | wc -l
|
||||
Reference in New Issue
Block a user