ModuFlow/mask-ddpm
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add comprehensive evaluation and ablation summaries

Browse Source
This commit is contained in:
mzYang
2026-03-27 22:11:07 +08:00
parent 17f69f963f
commit b8696d0c54
46 changed files with 30666 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

538
example/results/ablations/runs/eps_target__seed1337/comprehensive_eval.json Normal file
View File

@@ -0,0 +1,538 @@
{
"generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/eps_target__seed1337/generated.csv",
"reference_paths": [
"/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz",
"/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz",
"/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz"
],
"test_paths": [
"/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz",
"/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz",
"/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz",
"/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz",
"/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz"
],
"seq_len": 96,
"stride": 96,
"counts": {
"train_windows": 1024,
"generated_windows": 4,
"test_windows": 1024,
"test_anomalous_windows": 44,
"test_normal_windows": 980
},
"basic_eval": {
"avg_ks": 0.48224064465408806,
"avg_jsd": 0.10238307146791868,
"avg_lag1_diff": 0.7276772745273601
},
"two_sample": {
"continuous_mmd_rbf": 0.4687999486923218,
"continuous_mmd_gamma": 1.0991056714539588e-05,
"histogram_mmd_rbf": 0.46879076957702637,
"histogram_mmd_gamma": 1.0990790576152698e-05,
"discriminative_accuracy": 1.0,
"discriminative_balanced_accuracy": 0.5,
"discriminative_auroc": NaN
},
"diversity_privacy": {
"duplicate_rate": 0.0,
"exact_match_rate_to_train": 0.0,
"nn_gen_to_train_mean": 195.676513671875,
"nn_holdout_to_train_mean": 55.17637252807617,
"memorization_ratio": 3.5463823500232126,
"one_nn_two_sample_accuracy": 1.0
},
"coupling": {
"corr_mean_abs_diff": 0.1948104053735733,
"corr_frobenius": 15.79874038696289,
"lag1_corr_mean_abs_diff": 0.18509823083877563,
"lag1_corr_frobenius": 16.390331268310547,
"by_process": {
"P1": {
"corr_mean_abs_diff": 0.32595741748809814,
"corr_frobenius": 11.500425338745117,
"lag1_corr_mean_abs_diff": 0.35287562012672424,
"lag1_corr_frobenius": 12.468684196472168
},
"P2": {
"corr_mean_abs_diff": 0.11868169903755188,
"corr_frobenius": 2.282080888748169,
"lag1_corr_mean_abs_diff": 0.19088982045650482,
"lag1_corr_frobenius": 3.5559701919555664
},
"P3": {
"corr_mean_abs_diff": 0.33685311675071716,
"corr_frobenius": 2.3411834239959717,
"lag1_corr_mean_abs_diff": 0.5530110001564026,
"lag1_corr_frobenius": 3.203065872192383
},
"P4": {
"corr_mean_abs_diff": 0.29687002301216125,
"corr_frobenius": 4.520476818084717,
"lag1_corr_mean_abs_diff": 0.3345119059085846,
"lag1_corr_frobenius": 4.801176071166992
}
}
},
"frequency": {
"avg_psd_l1": 0.026633866131305695,
"avg_psd_cosine": 0.5700032617978024,
"avg_low_high_ratio_abs_diff": 7547185.2531314045
},
"type_metrics": {
"type1_program": {
"features": [
"P1_B4002",
"P2_MSD",
"P4_HT_LD",
"P1_B2004",
"P1_B3004",
"P1_B4022",
"P1_B3005"
],
"generated": {
"P1_B4002": {
"num_changes": 0.0,
"mean_dwell": 384.0,
"median_dwell": 384.0,
"mean_step": NaN,
"median_step": NaN
},
"P2_MSD": {
"num_changes": 0.0,
"mean_dwell": 384.0,
"median_dwell": 384.0,
"mean_step": NaN,
"median_step": NaN
},
"P4_HT_LD": {
"num_changes": 15.0,
"mean_dwell": 24.0,
"median_dwell": 2.5,
"mean_step": 68.42660395304362,
"median_step": 1.9043693542480469
},
"P1_B2004": {
"num_changes": 0.0,
"mean_dwell": 384.0,
"median_dwell": 384.0,
"mean_step": NaN,
"median_step": NaN
},
"P1_B3004": {
"num_changes": 0.0,
"mean_dwell": 384.0,
"median_dwell": 384.0,
"mean_step": NaN,
"median_step": NaN
},
"P1_B4022": {
"num_changes": 383.0,
"mean_dwell": 1.0,
"median_dwell": 1.0,
"mean_step": 0.01115664489586733,
"median_step": 0.004810333251953125
},
"P1_B3005": {
"num_changes": 0.0,
"mean_dwell": 384.0,
"median_dwell": 384.0,
"mean_step": NaN,
"median_step": NaN
}
},
"reference": {
"P1_B4002": {
"num_changes": 97.0,
"mean_dwell": 1003.1020408163265,
"median_dwell": 1.0,
"mean_step": 0.032477270696581025,
"median_step": 0.0126495361328125
},
"P2_MSD": {
"num_changes": 0.0,
"mean_dwell": 98304.0,
"median_dwell": 98304.0,
"mean_step": NaN,
"median_step": NaN
},
"P4_HT_LD": {
"num_changes": 63727.0,
"mean_dwell": 1.5425558624152649,
"median_dwell": 1.0,
"mean_step": 1.0409184914880636,
"median_step": 0.245941162109375
},
"P1_B2004": {
"num_changes": 145.0,
"mean_dwell": 673.3150684931506,
"median_dwell": 1.0,
"mean_step": 0.0015369655757114806,
"median_step": 0.00010000169277191162
},
"P1_B3004": {
"num_changes": 146.0,
"mean_dwell": 668.734693877551,
"median_dwell": 1.0,
"mean_step": 1.113193877755779,
"median_step": 0.819305419921875
},
"P1_B4022": {
"num_changes": 97122.0,
"mean_dwell": 1.0121598385552342,
"median_dwell": 1.0,
"mean_step": 0.011706427666086412,
"median_step": 0.00952911376953125
},
"P1_B3005": {
"num_changes": 148.0,
"mean_dwell": 659.7583892617449,
"median_dwell": 1.0,
"mean_step": 5.635673316749367,
"median_step": 5.34417724609375
}
},
"aggregates": {
"mean_abs_diff_num_changes": 22998.14285714286,
"mean_abs_diff_mean_dwell": 14201.625685203559,
"mean_abs_diff_median_dwell": 14207.642857142857,
"mean_abs_diff_mean_step": 33.693117622162895,
"mean_abs_diff_median_step": 0.831573486328125
}
},
"type2_controller": {
"features": [
"P1_B4005"
],
"generated": {
"P1_B4005": {
"saturation_ratio": 0.4140625,
"change_rate": 0.5169712793733682,
"step_median": 54.905609130859375
}
},
"reference": {
"P1_B4005": {
"saturation_ratio": 0.588958740234375,
"change_rate": 0.4608201173921447,
"step_median": 0.03202030062675476
}
},
"aggregates": {
"mean_abs_diff_saturation_ratio": 0.174896240234375,
"mean_abs_diff_change_rate": 0.056151161981223496,
"mean_abs_diff_step_median": 54.87358883023262
}
},
"type3_actuator": {
"features": [
"P1_PCV02Z",
"P1_PCV01Z",
"P1_PCV01D",
"P1_FCV02Z",
"P1_FCV03D",
"P1_FCV03Z",
"P1_LCV01D",
"P1_LCV01Z"
],
"generated": {
"P1_PCV02Z": {
"unique_ratio": 0.020833333333333332,
"top1_mass": 0.3151041666666667,
"top3_mass": 0.7630208333333334,
"median_dwell": 1.0
},
"P1_PCV01Z": {
"unique_ratio": 0.19270833333333334,
"top1_mass": 0.4401041666666667,
"top3_mass": 0.8072916666666666,
"median_dwell": 1.0
},
"P1_PCV01D": {
"unique_ratio": 0.15625,
"top1_mass": 0.5026041666666666,
"top3_mass": 0.8489583333333334,
"median_dwell": 1.0
},
"P1_FCV02Z": {
"unique_ratio": 0.13802083333333334,
"top1_mass": 0.2421875,
"top3_mass": 0.4453125,
"median_dwell": 1.0
},
"P1_FCV03D": {
"unique_ratio": 0.4348958333333333,
"top1_mass": 0.5442708333333334,
"top3_mass": 0.5598958333333334,
"median_dwell": 1.0
},
"P1_FCV03Z": {
"unique_ratio": 0.3671875,
"top1_mass": 0.28125,
"top3_mass": 0.3203125,
"median_dwell": 1.0
},
"P1_LCV01D": {
"unique_ratio": 0.4713541666666667,
"top1_mass": 0.4817708333333333,
"top3_mass": 0.4921875,
"median_dwell": 1.0
},
"P1_LCV01Z": {
"unique_ratio": 0.2682291666666667,
"top1_mass": 0.2838541666666667,
"top3_mass": 0.3671875,
"median_dwell": 1.0
}
},
"reference": {
"P1_PCV02Z": {
"unique_ratio": 5.0862630208333336e-05,
"top1_mass": 0.5122273763020834,
"top3_mass": 0.9970194498697916,
"median_dwell": 2.0
},
"P1_PCV01Z": {
"unique_ratio": 0.063232421875,
"top1_mass": 0.009714762369791666,
"top3_mass": 0.020904541015625,
"median_dwell": 1.0
},
"P1_PCV01D": {
"unique_ratio": 0.074310302734375,
"top1_mass": 0.043843587239583336,
"top3_mass": 0.044850667317708336,
"median_dwell": 1.0
},
"P1_FCV02Z": {
"unique_ratio": 0.058146158854166664,
"top1_mass": 0.20765177408854166,
"top3_mass": 0.3501078287760417,
"median_dwell": 1.0
},
"P1_FCV03D": {
"unique_ratio": 0.017527262369791668,
"top1_mass": 0.004384358723958333,
"top3_mass": 0.012776692708333334,
"median_dwell": 1.0
},
"P1_FCV03Z": {
"unique_ratio": 0.013102213541666666,
"top1_mass": 0.10939534505208333,
"top3_mass": 0.2132568359375,
"median_dwell": 1.0
},
"P1_LCV01D": {
"unique_ratio": 0.015523274739583334,
"top1_mass": 0.0033976236979166665,
"top3_mass": 0.009999593098958334,
"median_dwell": 1.0
},
"P1_LCV01Z": {
"unique_ratio": 0.012715657552083334,
"top1_mass": 0.047566731770833336,
"top3_mass": 0.1214599609375,
"median_dwell": 1.0
}
},
"aggregates": {
"mean_abs_diff_unique_ratio": 0.22435887654622397,
"mean_abs_diff_top1_mass": 0.3184013366699219,
"mean_abs_diff_top3_mass": 0.4127235412597656,
"mean_abs_diff_median_dwell": 0.125
}
},
"type4_pv": {
"features": [
"P1_PIT02",
"P2_SIT02",
"P1_FT03"
],
"generated": {
"P1_PIT02": {
"q05": 0.19251999258995056,
"q50": 0.27557000517845154,
"q95": 0.42587000131607056,
"tail_ratio": 1.809752839922259
},
"P2_SIT02": {
"q05": 774.8785400390625,
"q50": 784.5530395507812,
"q95": 786.7108764648438,
"tail_ratio": 0.22304377724643076
},
"P1_FT03": {
"q05": 187.14901733398438,
"q50": 298.9196472167969,
"q95": 316.24639892578125,
"tail_ratio": 0.15502061433447098
}
},
"reference": {
"P1_PIT02": {
"q05": 0.20232999324798584,
"q50": 0.2763400077819824,
"q95": 1.8746999502182007,
"tail_ratio": 21.596535989085773
},
"P2_SIT02": {
"q05": 771.5216674804688,
"q50": 779.1593017578125,
"q95": 785.1046752929688,
"tail_ratio": 0.7784312942022615
},
"P1_FT03": {
"q05": 201.6184539794922,
"q50": 300.2251892089844,
"q95": 313.76690673828125,
"tail_ratio": 0.13733055351422585
}
},
"aggregates": {
"mean_abs_diff_q05": 5.9453730682531996,
"mean_abs_diff_q50": 2.23334992925326,
"mean_abs_diff_q95": 1.8448411027590434,
"mean_abs_diff_tail_ratio": 6.786620242313197
}
},
"type5_program_proxy": {
"features": [
"P1_FT03Z",
"P1_FT02Z"
],
"generated": {
"P1_FT03Z": {
"num_changes": 324.0,
"mean_dwell": 1.1815384615384616,
"median_dwell": 1.0,
"mean_step": 65.06421595443913,
"median_step": 105.447021484375
},
"P1_FT02Z": {
"num_changes": 319.0,
"mean_dwell": 1.2,
"median_dwell": 1.0,
"mean_step": 1047.0514582421713,
"median_step": 1134.979736328125
}
},
"reference": {
"P1_FT03Z": {
"num_changes": 83889.0,
"mean_dwell": 1.1718202407915126,
"median_dwell": 1.0,
"mean_step": 0.9289188007355237,
"median_step": 0.6378173828125
},
"P1_FT02Z": {
"num_changes": 95436.0,
"mean_dwell": 1.0300407598730053,
"median_dwell": 1.0,
"mean_step": 14.373282350295296,
"median_step": 5.89599609375
}
},
"aggregates": {
"mean_abs_diff_num_changes": 89341.0,
"mean_abs_diff_mean_dwell": 0.08983873043697188,
"mean_abs_diff_median_dwell": 0.0,
"mean_abs_diff_mean_step": 548.4067365227897,
"mean_abs_diff_median_step": 616.9464721679688
}
},
"type6_aux": {
"features": [
"P4_HT_PO",
"P2_24Vdc",
"P2_HILout"
],
"generated": {
"P4_HT_PO": {
"mean": 33.98807907104492,
"std": 31.542545318603516,
"lag1": 0.057515524327754974
},
"P2_24Vdc": {
"mean": 28.0284481048584,
"std": 0.0041008396074175835,
"lag1": 0.011104353703558445
},
"P2_HILout": {
"mean": 708.3784790039062,
"std": 16.135107040405273,
"lag1": 0.22598320245742798
}
},
"reference": {
"P4_HT_PO": {
"mean": 34.35928726196289,
"std": 31.320688247680664,
"lag1": 0.9990671873092651
},
"P2_24Vdc": {
"mean": 28.0284366607666,
"std": 0.0032698388677090406,
"lag1": 0.05670436844229698
},
"P2_HILout": {
"mean": 711.9729614257812,
"std": 8.238578796386719,
"lag1": 0.2585394084453583
}
},
"aggregates": {
"mean_abs_diff_mean": 1.3219006856282551,
"mean_abs_diff_std": 2.7064054385603717,
"mean_abs_diff_lag1": 0.33990262790272635
}
}
},
"predictive_consistency": {
"real_only": {
"rmse": 0.5583509206771851,
"mae": 0.366405189037323
},
"synthetic_only": {
"rmse": 0.9675812721252441,
"mae": 0.7771188616752625
},
"real_plus_synthetic": {
"rmse": 0.5489299893379211,
"mae": 0.3588286340236664
},
"rmse_ratio_synth_to_real": 1.7329267962014498
},
"anomaly_utility": {
"real_only": {
"auroc": 0.8881493506493506,
"auprc": 0.6527744226377132,
"threshold": 0.3682355284690857,
"f1": 0.35714285714285715,
"precision": 0.24193548387096775,
"recall": 0.6818181818181818,
"best_f1": 0.763157894736842,
"best_f1_threshold": 0.49033087491989136
},
"synthetic_only": {
"auroc": 0.8216141001855286,
"auprc": 0.5817769935282933,
"threshold": 0.0175539031624794,
"f1": 0.08239700374531835,
"precision": 0.04296875,
"recall": 1.0,
"best_f1": 0.6666666666666667,
"best_f1_threshold": 1.498559594154358
},
"real_plus_synthetic": {
"auroc": 0.8842300556586271,
"auprc": 0.6466204571725106,
"threshold": 0.3626948595046997,
"f1": 0.3372093023255814,
"precision": 0.2265625,
"recall": 0.6590909090909091,
"best_f1": 0.763157894736842,
"best_f1_threshold": 0.4720713794231415
}
}
}