From b8696d0c5472f4106c13045cadebda5e09bb56b6 Mon Sep 17 00:00:00 2001 From: mzYang Date: Fri, 27 Mar 2026 22:11:07 +0800 Subject: [PATCH] Add comprehensive evaluation and ablation summaries --- .../results/ablations/ablation_summary.csv | 5 + .../results/ablations/ablation_summary.json | 97 ++ .../results/ablations/benchmark_history.csv | 12 + .../results/ablations/benchmark_summary.csv | 11 + .../comprehensive_eval.json | 538 ++++++++++ .../comprehensive_eval_post.json | 538 ++++++++++ .../runs/eps_target__seed1337/eval.json | 974 ++++++++++++++++++ .../runs/eps_target__seed1337/eval_post.json | 971 +++++++++++++++++ .../full__seed1337/comprehensive_eval.json | 537 ++++++++++ .../comprehensive_eval_post.json | 538 ++++++++++ .../ablations/runs/full__seed1337/eval.json | 974 ++++++++++++++++++ .../runs/full__seed1337/eval_post.json | 971 +++++++++++++++++ .../comprehensive_eval.json | 538 ++++++++++ .../comprehensive_eval_post.json | 538 ++++++++++ .../no_file_condition__seed1337/eval.json | 974 ++++++++++++++++++ .../eval_post.json | 971 +++++++++++++++++ .../comprehensive_eval.json | 538 ++++++++++ .../comprehensive_eval_post.json | 538 ++++++++++ .../no_post_calibration__seed1337/eval.json | 974 ++++++++++++++++++ .../eval_post.json | 971 +++++++++++++++++ .../comprehensive_eval.json | 538 ++++++++++ .../comprehensive_eval_post.json | 538 ++++++++++ .../runs/no_quantile__seed1337/eval.json | 974 ++++++++++++++++++ .../runs/no_quantile__seed1337/eval_post.json | 971 +++++++++++++++++ .../comprehensive_eval.json | 537 ++++++++++ .../comprehensive_eval_post.json | 538 ++++++++++ .../runs/no_quantile_loss__seed1337/eval.json | 974 ++++++++++++++++++ .../no_quantile_loss__seed1337/eval_post.json | 971 +++++++++++++++++ .../comprehensive_eval.json | 537 ++++++++++ .../comprehensive_eval_post.json | 538 ++++++++++ .../runs/no_residual_stat__seed1337/eval.json | 974 ++++++++++++++++++ .../no_residual_stat__seed1337/eval_post.json | 971 +++++++++++++++++ .../comprehensive_eval.json | 537 ++++++++++ .../comprehensive_eval_post.json | 538 ++++++++++ .../runs/no_snr_weight__seed1337/eval.json | 974 ++++++++++++++++++ .../no_snr_weight__seed1337/eval_post.json | 971 +++++++++++++++++ .../comprehensive_eval.json | 538 ++++++++++ .../comprehensive_eval_post.json | 538 ++++++++++ .../runs/no_temporal__seed1337/eval.json | 974 ++++++++++++++++++ .../runs/no_temporal__seed1337/eval_post.json | 971 +++++++++++++++++ .../comprehensive_eval.json | 168 +++ .../comprehensive_eval_post.json | 168 +++ .../runs/no_type_routing__seed1337/eval.json | 974 ++++++++++++++++++ .../no_type_routing__seed1337/eval_post.json | 971 +++++++++++++++++ example/results/comprehensive_eval.json | 537 ++++++++++ example/results/comprehensive_eval_post.json | 538 ++++++++++ 46 files changed, 30666 insertions(+) create mode 100644 example/results/ablations/ablation_summary.csv create mode 100644 example/results/ablations/ablation_summary.json create mode 100644 example/results/ablations/benchmark_history.csv create mode 100644 example/results/ablations/benchmark_summary.csv create mode 100644 example/results/ablations/runs/eps_target__seed1337/comprehensive_eval.json create mode 100644 example/results/ablations/runs/eps_target__seed1337/comprehensive_eval_post.json create mode 100644 example/results/ablations/runs/eps_target__seed1337/eval.json create mode 100644 example/results/ablations/runs/eps_target__seed1337/eval_post.json create mode 100644 example/results/ablations/runs/full__seed1337/comprehensive_eval.json create mode 100644 example/results/ablations/runs/full__seed1337/comprehensive_eval_post.json create mode 100644 example/results/ablations/runs/full__seed1337/eval.json create mode 100644 example/results/ablations/runs/full__seed1337/eval_post.json create mode 100644 example/results/ablations/runs/no_file_condition__seed1337/comprehensive_eval.json create mode 100644 example/results/ablations/runs/no_file_condition__seed1337/comprehensive_eval_post.json create mode 100644 example/results/ablations/runs/no_file_condition__seed1337/eval.json create mode 100644 example/results/ablations/runs/no_file_condition__seed1337/eval_post.json create mode 100644 example/results/ablations/runs/no_post_calibration__seed1337/comprehensive_eval.json create mode 100644 example/results/ablations/runs/no_post_calibration__seed1337/comprehensive_eval_post.json create mode 100644 example/results/ablations/runs/no_post_calibration__seed1337/eval.json create mode 100644 example/results/ablations/runs/no_post_calibration__seed1337/eval_post.json create mode 100644 example/results/ablations/runs/no_quantile__seed1337/comprehensive_eval.json create mode 100644 example/results/ablations/runs/no_quantile__seed1337/comprehensive_eval_post.json create mode 100644 example/results/ablations/runs/no_quantile__seed1337/eval.json create mode 100644 example/results/ablations/runs/no_quantile__seed1337/eval_post.json create mode 100644 example/results/ablations/runs/no_quantile_loss__seed1337/comprehensive_eval.json create mode 100644 example/results/ablations/runs/no_quantile_loss__seed1337/comprehensive_eval_post.json create mode 100644 example/results/ablations/runs/no_quantile_loss__seed1337/eval.json create mode 100644 example/results/ablations/runs/no_quantile_loss__seed1337/eval_post.json create mode 100644 example/results/ablations/runs/no_residual_stat__seed1337/comprehensive_eval.json create mode 100644 example/results/ablations/runs/no_residual_stat__seed1337/comprehensive_eval_post.json create mode 100644 example/results/ablations/runs/no_residual_stat__seed1337/eval.json create mode 100644 example/results/ablations/runs/no_residual_stat__seed1337/eval_post.json create mode 100644 example/results/ablations/runs/no_snr_weight__seed1337/comprehensive_eval.json create mode 100644 example/results/ablations/runs/no_snr_weight__seed1337/comprehensive_eval_post.json create mode 100644 example/results/ablations/runs/no_snr_weight__seed1337/eval.json create mode 100644 example/results/ablations/runs/no_snr_weight__seed1337/eval_post.json create mode 100644 example/results/ablations/runs/no_temporal__seed1337/comprehensive_eval.json create mode 100644 example/results/ablations/runs/no_temporal__seed1337/comprehensive_eval_post.json create mode 100644 example/results/ablations/runs/no_temporal__seed1337/eval.json create mode 100644 example/results/ablations/runs/no_temporal__seed1337/eval_post.json create mode 100644 example/results/ablations/runs/no_type_routing__seed1337/comprehensive_eval.json create mode 100644 example/results/ablations/runs/no_type_routing__seed1337/comprehensive_eval_post.json create mode 100644 example/results/ablations/runs/no_type_routing__seed1337/eval.json create mode 100644 example/results/ablations/runs/no_type_routing__seed1337/eval_post.json create mode 100644 example/results/comprehensive_eval.json create mode 100644 example/results/comprehensive_eval_post.json diff --git a/example/results/ablations/ablation_summary.csv b/example/results/ablations/ablation_summary.csv new file mode 100644 index 0000000..d08676c --- /dev/null +++ b/example/results/ablations/ablation_summary.csv @@ -0,0 +1,5 @@ +avg_jsd,avg_ks,avg_lag1_diff,avg_psd_l1,continuous_mmd_rbf,corr_mean_abs_diff,discriminative_accuracy,memorization_ratio,post_avg_jsd,post_avg_ks,post_avg_lag1_diff,predictive_rmse_real,predictive_rmse_synth,run_dir,seed,utility_auprc_aug,utility_auprc_real,utility_auprc_synth,variant +0.02154500462890076,0.39950668238993714,0.2994253248181626,0.019868340343236923,0.6485295295715332,0.21394668519496918,1.0,2.98201284485682,,,,0.5583509206771851,0.9607057571411133,/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_snr_weight__seed1337,1337,0.637313246257706,0.6527744226377132,0.5839558451818384,no_snr_weight +0.017899328657243745,0.4130987421383648,0.3108961322076135,0.02026984840631485,0.6504309177398682,0.21265330910682678,1.0,2.948979736458853,,,,0.5583509206771851,0.9654670357704163,/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_quantile_loss__seed1337,1337,0.6451654579890704,0.6527744226377132,0.5904586531962263,no_quantile_loss +0.02930392024585336,0.4039510220125786,0.28531566380969187,0.019299142062664032,0.6561890840530396,0.21048887073993683,1.0,2.953516468815496,,,,0.5583509206771851,0.9696047902107239,/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_residual_stat__seed1337,1337,0.6465074146607899,0.6527744226377132,0.5848963274786471,no_residual_stat +0.10238307146791868,0.48224064465408806,0.7276772745273601,0.026633866131305695,0.4687999486923218,0.1948104053735733,1.0,3.5463823500232126,,,,0.5583509206771851,0.9675812721252441,/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/eps_target__seed1337,1337,0.6466204571725106,0.6527744226377132,0.5817769935282933,eps_target diff --git a/example/results/ablations/ablation_summary.json b/example/results/ablations/ablation_summary.json new file mode 100644 index 0000000..1782ff1 --- /dev/null +++ b/example/results/ablations/ablation_summary.json @@ -0,0 +1,97 @@ +{ + "variants": [ + "no_snr_weight", + "no_quantile_loss", + "no_residual_stat", + "eps_target" + ], + "seeds": [ + 1337 + ], + "rows": [ + { + "variant": "no_snr_weight", + "seed": 1337, + "run_dir": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_snr_weight__seed1337", + "avg_ks": 0.39950668238993714, + "avg_jsd": 0.02154500462890076, + "avg_lag1_diff": 0.2994253248181626, + "continuous_mmd_rbf": 0.6485295295715332, + "discriminative_accuracy": 1.0, + "corr_mean_abs_diff": 0.21394668519496918, + "avg_psd_l1": 0.019868340343236923, + "memorization_ratio": 2.98201284485682, + "predictive_rmse_real": 0.5583509206771851, + "predictive_rmse_synth": 0.9607057571411133, + "utility_auprc_real": 0.6527744226377132, + "utility_auprc_synth": 0.5839558451818384, + "utility_auprc_aug": 0.637313246257706, + "post_avg_ks": null, + "post_avg_jsd": null, + "post_avg_lag1_diff": null + }, + { + "variant": "no_quantile_loss", + "seed": 1337, + "run_dir": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_quantile_loss__seed1337", + "avg_ks": 0.4130987421383648, + "avg_jsd": 0.017899328657243745, + "avg_lag1_diff": 0.3108961322076135, + "continuous_mmd_rbf": 0.6504309177398682, + "discriminative_accuracy": 1.0, + "corr_mean_abs_diff": 0.21265330910682678, + "avg_psd_l1": 0.02026984840631485, + "memorization_ratio": 2.948979736458853, + "predictive_rmse_real": 0.5583509206771851, + "predictive_rmse_synth": 0.9654670357704163, + "utility_auprc_real": 0.6527744226377132, + "utility_auprc_synth": 0.5904586531962263, + "utility_auprc_aug": 0.6451654579890704, + "post_avg_ks": null, + "post_avg_jsd": null, + "post_avg_lag1_diff": null + }, + { + "variant": "no_residual_stat", + "seed": 1337, + "run_dir": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_residual_stat__seed1337", + "avg_ks": 0.4039510220125786, + "avg_jsd": 0.02930392024585336, + "avg_lag1_diff": 0.28531566380969187, + "continuous_mmd_rbf": 0.6561890840530396, + "discriminative_accuracy": 1.0, + "corr_mean_abs_diff": 0.21048887073993683, + "avg_psd_l1": 0.019299142062664032, + "memorization_ratio": 2.953516468815496, + "predictive_rmse_real": 0.5583509206771851, + "predictive_rmse_synth": 0.9696047902107239, + "utility_auprc_real": 0.6527744226377132, + "utility_auprc_synth": 0.5848963274786471, + "utility_auprc_aug": 0.6465074146607899, + "post_avg_ks": null, + "post_avg_jsd": null, + "post_avg_lag1_diff": null + }, + { + "variant": "eps_target", + "seed": 1337, + "run_dir": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/eps_target__seed1337", + "avg_ks": 0.48224064465408806, + "avg_jsd": 0.10238307146791868, + "avg_lag1_diff": 0.7276772745273601, + "continuous_mmd_rbf": 0.4687999486923218, + "discriminative_accuracy": 1.0, + "corr_mean_abs_diff": 0.1948104053735733, + "avg_psd_l1": 0.026633866131305695, + "memorization_ratio": 3.5463823500232126, + "predictive_rmse_real": 0.5583509206771851, + "predictive_rmse_synth": 0.9675812721252441, + "utility_auprc_real": 0.6527744226377132, + "utility_auprc_synth": 0.5817769935282933, + "utility_auprc_aug": 0.6466204571725106, + "post_avg_ks": null, + "post_avg_jsd": null, + "post_avg_lag1_diff": null + } + ] +} \ No newline at end of file diff --git a/example/results/ablations/benchmark_history.csv b/example/results/ablations/benchmark_history.csv new file mode 100644 index 0000000..99e3379 --- /dev/null +++ b/example/results/ablations/benchmark_history.csv @@ -0,0 +1,12 @@ +timestamp,run_name,config,seed,avg_ks,avg_jsd,avg_lag1_diff +2026-03-26T15:45:21.369886,full__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/full.json,1337,0.40165841194968555,0.028006235870934912,0.29136546564037447 +2026-03-26T16:53:34.911090,no_temporal__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_temporal.json,1337,0.40846108490566035,0.031160008477122598,0.6642869349466621 +2026-03-26T17:34:38.356691,no_quantile__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_quantile.json,1337,0.5986268081761006,0.010030341725636,0.15569708276765754 +2026-03-26T18:23:33.132954,no_post_calibration__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_post_calibration.json,1337,0.5430239779874214,0.024122590895914533,0.25321129348110133 +2026-03-26T19:34:31.196915,no_file_condition__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_file_condition.json,1337,0.4049190251572327,0.0327003560472195,0.23744482453266894 +2026-03-26T20:46:45.741433,no_type_routing__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_type_routing.json,1337,0.35632916666666664,0.021975792751969963,0.13777518542478578 +2026-03-27T06:51:27.544936,no_snr_weight__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_snr_weight.json,1337,0.3990649371069182,0.024572602997555995,0.3062329530249939 +2026-03-27T08:17:16.994922,no_snr_weight__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_snr_weight.json,1337,0.39950668238993714,0.02154500462890076,0.2994253248181626 +2026-03-27T09:27:17.154215,no_quantile_loss__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_quantile_loss.json,1337,0.4130987421383648,0.017899328657243745,0.3108961322076135 +2026-03-27T10:37:15.958652,no_residual_stat__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_residual_stat.json,1337,0.4039510220125786,0.02930392024585336,0.28531566380969187 +2026-03-27T11:48:13.637772,eps_target__seed1337,/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/eps_target.json,1337,0.48224064465408806,0.10238307146791868,0.7276772745273601 diff --git a/example/results/ablations/benchmark_summary.csv b/example/results/ablations/benchmark_summary.csv new file mode 100644 index 0000000..f5f2c46 --- /dev/null +++ b/example/results/ablations/benchmark_summary.csv @@ -0,0 +1,11 @@ +config,n_runs,avg_ks_mean,avg_ks_std,avg_jsd_mean,avg_jsd_std,avg_lag1_diff_mean,avg_lag1_diff_std,best_run_name,best_avg_ks +/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/eps_target.json,1,0.48224064465408806,0.0,0.10238307146791868,0.0,0.7276772745273601,0.0,eps_target__seed1337,0.48224064465408806 +/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/full.json,1,0.40165841194968555,0.0,0.028006235870934912,0.0,0.29136546564037447,0.0,full__seed1337,0.40165841194968555 +/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_file_condition.json,1,0.4049190251572327,0.0,0.0327003560472195,0.0,0.23744482453266894,0.0,no_file_condition__seed1337,0.4049190251572327 +/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_post_calibration.json,1,0.5430239779874214,0.0,0.024122590895914533,0.0,0.25321129348110133,0.0,no_post_calibration__seed1337,0.5430239779874214 +/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_quantile.json,1,0.5986268081761006,0.0,0.010030341725636,0.0,0.15569708276765754,0.0,no_quantile__seed1337,0.5986268081761006 +/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_quantile_loss.json,1,0.4130987421383648,0.0,0.017899328657243745,0.0,0.3108961322076135,0.0,no_quantile_loss__seed1337,0.4130987421383648 +/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_residual_stat.json,1,0.4039510220125786,0.0,0.02930392024585336,0.0,0.28531566380969187,0.0,no_residual_stat__seed1337,0.4039510220125786 +/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_snr_weight.json,2,0.39928580974842764,0.00031236108517985957,0.023058803813228376,0.002140835337185445,0.30282913892157826,0.004813720068847257,no_snr_weight__seed1337,0.3990649371069182 +/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_temporal.json,1,0.40846108490566035,0.0,0.031160008477122598,0.0,0.6642869349466621,0.0,no_temporal__seed1337,0.40846108490566035 +/root/autodl-tmp/mask-ddpm/example/results/ablations/configs/no_type_routing.json,1,0.35632916666666664,0.0,0.021975792751969963,0.0,0.13777518542478578,0.0,no_type_routing__seed1337,0.35632916666666664 diff --git a/example/results/ablations/runs/eps_target__seed1337/comprehensive_eval.json b/example/results/ablations/runs/eps_target__seed1337/comprehensive_eval.json new file mode 100644 index 0000000..5b98666 --- /dev/null +++ b/example/results/ablations/runs/eps_target__seed1337/comprehensive_eval.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/eps_target__seed1337/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.48224064465408806, + "avg_jsd": 0.10238307146791868, + "avg_lag1_diff": 0.7276772745273601 + }, + "two_sample": { + "continuous_mmd_rbf": 0.4687999486923218, + "continuous_mmd_gamma": 1.0991056714539588e-05, + "histogram_mmd_rbf": 0.46879076957702637, + "histogram_mmd_gamma": 1.0990790576152698e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 195.676513671875, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 3.5463823500232126, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.1948104053735733, + "corr_frobenius": 15.79874038696289, + "lag1_corr_mean_abs_diff": 0.18509823083877563, + "lag1_corr_frobenius": 16.390331268310547, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.32595741748809814, + "corr_frobenius": 11.500425338745117, + "lag1_corr_mean_abs_diff": 0.35287562012672424, + "lag1_corr_frobenius": 12.468684196472168 + }, + "P2": { + "corr_mean_abs_diff": 0.11868169903755188, + "corr_frobenius": 2.282080888748169, + "lag1_corr_mean_abs_diff": 0.19088982045650482, + "lag1_corr_frobenius": 3.5559701919555664 + }, + "P3": { + "corr_mean_abs_diff": 0.33685311675071716, + "corr_frobenius": 2.3411834239959717, + "lag1_corr_mean_abs_diff": 0.5530110001564026, + "lag1_corr_frobenius": 3.203065872192383 + }, + "P4": { + "corr_mean_abs_diff": 0.29687002301216125, + "corr_frobenius": 4.520476818084717, + "lag1_corr_mean_abs_diff": 0.3345119059085846, + "lag1_corr_frobenius": 4.801176071166992 + } + } + }, + "frequency": { + "avg_psd_l1": 0.026633866131305695, + "avg_psd_cosine": 0.5700032617978024, + "avg_low_high_ratio_abs_diff": 7547185.2531314045 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 15.0, + "mean_dwell": 24.0, + "median_dwell": 2.5, + "mean_step": 68.42660395304362, + "median_step": 1.9043693542480469 + }, + "P1_B2004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B3004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.01115664489586733, + "median_step": 0.004810333251953125 + }, + "P1_B3005": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22998.14285714286, + "mean_abs_diff_mean_dwell": 14201.625685203559, + "mean_abs_diff_median_dwell": 14207.642857142857, + "mean_abs_diff_mean_step": 33.693117622162895, + "mean_abs_diff_median_step": 0.831573486328125 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.4140625, + "change_rate": 0.5169712793733682, + "step_median": 54.905609130859375 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.174896240234375, + "mean_abs_diff_change_rate": 0.056151161981223496, + "mean_abs_diff_step_median": 54.87358883023262 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.020833333333333332, + "top1_mass": 0.3151041666666667, + "top3_mass": 0.7630208333333334, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.19270833333333334, + "top1_mass": 0.4401041666666667, + "top3_mass": 0.8072916666666666, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.15625, + "top1_mass": 0.5026041666666666, + "top3_mass": 0.8489583333333334, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.13802083333333334, + "top1_mass": 0.2421875, + "top3_mass": 0.4453125, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.4348958333333333, + "top1_mass": 0.5442708333333334, + "top3_mass": 0.5598958333333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.3671875, + "top1_mass": 0.28125, + "top3_mass": 0.3203125, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.4713541666666667, + "top1_mass": 0.4817708333333333, + "top3_mass": 0.4921875, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.2682291666666667, + "top1_mass": 0.2838541666666667, + "top3_mass": 0.3671875, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.22435887654622397, + "mean_abs_diff_top1_mass": 0.3184013366699219, + "mean_abs_diff_top3_mass": 0.4127235412597656, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.19251999258995056, + "q50": 0.27557000517845154, + "q95": 0.42587000131607056, + "tail_ratio": 1.809752839922259 + }, + "P2_SIT02": { + "q05": 774.8785400390625, + "q50": 784.5530395507812, + "q95": 786.7108764648438, + "tail_ratio": 0.22304377724643076 + }, + "P1_FT03": { + "q05": 187.14901733398438, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.15502061433447098 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 5.9453730682531996, + "mean_abs_diff_q50": 2.23334992925326, + "mean_abs_diff_q95": 1.8448411027590434, + "mean_abs_diff_tail_ratio": 6.786620242313197 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 324.0, + "mean_dwell": 1.1815384615384616, + "median_dwell": 1.0, + "mean_step": 65.06421595443913, + "median_step": 105.447021484375 + }, + "P1_FT02Z": { + "num_changes": 319.0, + "mean_dwell": 1.2, + "median_dwell": 1.0, + "mean_step": 1047.0514582421713, + "median_step": 1134.979736328125 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89341.0, + "mean_abs_diff_mean_dwell": 0.08983873043697188, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 548.4067365227897, + "mean_abs_diff_median_step": 616.9464721679688 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 33.98807907104492, + "std": 31.542545318603516, + "lag1": 0.057515524327754974 + }, + "P2_24Vdc": { + "mean": 28.0284481048584, + "std": 0.0041008396074175835, + "lag1": 0.011104353703558445 + }, + "P2_HILout": { + "mean": 708.3784790039062, + "std": 16.135107040405273, + "lag1": 0.22598320245742798 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 1.3219006856282551, + "mean_abs_diff_std": 2.7064054385603717, + "mean_abs_diff_lag1": 0.33990262790272635 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9675812721252441, + "mae": 0.7771188616752625 + }, + "real_plus_synthetic": { + "rmse": 0.5489299893379211, + "mae": 0.3588286340236664 + }, + "rmse_ratio_synth_to_real": 1.7329267962014498 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8216141001855286, + "auprc": 0.5817769935282933, + "threshold": 0.0175539031624794, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6666666666666667, + "best_f1_threshold": 1.498559594154358 + }, + "real_plus_synthetic": { + "auroc": 0.8842300556586271, + "auprc": 0.6466204571725106, + "threshold": 0.3626948595046997, + "f1": 0.3372093023255814, + "precision": 0.2265625, + "recall": 0.6590909090909091, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.4720713794231415 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/eps_target__seed1337/comprehensive_eval_post.json b/example/results/ablations/runs/eps_target__seed1337/comprehensive_eval_post.json new file mode 100644 index 0000000..f9210c4 --- /dev/null +++ b/example/results/ablations/runs/eps_target__seed1337/comprehensive_eval_post.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/eps_target__seed1337/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.19920897483825684, + "continuous_mmd_gamma": 1.9479406766841545e-05, + "histogram_mmd_rbf": 0.19920587539672852, + "histogram_mmd_gamma": 1.947866865125894e-05, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 150.8397216796875, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 2.7337738015113913, + "one_nn_two_sample_accuracy": 0.9959349593495935 + }, + "coupling": { + "corr_mean_abs_diff": 0.18258368968963623, + "corr_frobenius": 15.465909957885742, + "lag1_corr_mean_abs_diff": 0.18536746501922607, + "lag1_corr_frobenius": 16.367128372192383, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.330845445394516, + "corr_frobenius": 11.653647422790527, + "lag1_corr_mean_abs_diff": 0.35905906558036804, + "lag1_corr_frobenius": 12.64593505859375 + }, + "P2": { + "corr_mean_abs_diff": 0.12455040216445923, + "corr_frobenius": 2.371154546737671, + "lag1_corr_mean_abs_diff": 0.19896332919597626, + "lag1_corr_frobenius": 3.6055963039398193 + }, + "P3": { + "corr_mean_abs_diff": 0.33685311675071716, + "corr_frobenius": 2.3411834239959717, + "lag1_corr_mean_abs_diff": 0.5530110001564026, + "lag1_corr_frobenius": 3.203065872192383 + }, + "P4": { + "corr_mean_abs_diff": 0.3033543527126312, + "corr_frobenius": 4.657331943511963, + "lag1_corr_mean_abs_diff": 0.3527607321739197, + "lag1_corr_frobenius": 5.017890930175781 + } + } + }, + "frequency": { + "avg_psd_l1": 0.02700667828321457, + "avg_psd_cosine": 0.5953845893437008, + "avg_low_high_ratio_abs_diff": 19.91622868380982 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 308.0, + "mean_dwell": 1.2427184466019416, + "median_dwell": 1.0, + "mean_step": 0.8413416007896523, + "median_step": 0.7274894714355469 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 347.0, + "mean_dwell": 1.103448275862069, + "median_dwell": 1.0, + "mean_step": 41.3141472878254, + "median_step": 54.116028785705566 + }, + "P1_B2004": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 0.014068839402653235, + "median_step": 0.0017450004816055298 + }, + "P1_B3004": { + "num_changes": 370.0, + "mean_dwell": 1.0350404312668464, + "median_dwell": 1.0, + "mean_step": 14.77256890374261, + "median_step": 10.804779052734375 + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.8157603410765646, + "median_step": 0.5944099426269531 + }, + "P1_B3005": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 64.28072175926926, + "median_step": 41.228179931640625 + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22997.85714285714, + "mean_abs_diff_mean_dwell": 14417.281142645239, + "mean_abs_diff_median_dwell": 13988.57142857143, + "mean_abs_diff_mean_step": 19.03385039702909, + "mean_abs_diff_median_step": 16.840154950817425 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6822916666666666, + "change_rate": 0.7702349869451697, + "step_median": 68.81373023986816 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.09333292643229163, + "mean_abs_diff_change_rate": 0.309414869553025, + "mean_abs_diff_step_median": 68.78170993924141 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.015625, + "top1_mass": 0.40625, + "top3_mass": 0.7760416666666666, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8776041666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.059895833333333336, + "top3_mass": 0.0703125, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.18229166666666666, + "top3_mass": 0.3697916666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.8463541666666666, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4817708333333333, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.7864583333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3515625, + "top1_mass": 0.036458333333333336, + "top3_mass": 0.09635416666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5303484598795573, + "mean_abs_diff_top1_mass": 0.03112411499023438, + "mean_abs_diff_top3_mass": 0.05629857381184896, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.19251999258995056, + "q50": 0.27557000517845154, + "q95": 0.42587000131607056, + "tail_ratio": 1.809752839922259 + }, + "P2_SIT02": { + "q05": 774.8785400390625, + "q50": 784.5530395507812, + "q95": 786.7108764648438, + "tail_ratio": 0.22304377724643076 + }, + "P1_FT03": { + "q05": 187.14901733398438, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.15502061433447098 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 5.9453730682531996, + "mean_abs_diff_q50": 2.23334992925326, + "mean_abs_diff_q95": 1.8448411027590434, + "mean_abs_diff_tail_ratio": 6.786620242313197 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 64.74391126130756, + "median_step": 62.410614013671875 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 1312.3134692358592, + "median_step": 1284.6439208984375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.09039718638964156, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 680.877589673068, + "mean_abs_diff_median_step": 670.2603607177734 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 35.254512786865234, + "std": 31.650897979736328, + "lag1": -0.0927804708480835 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.003297113813459873, + "lag1": 0.06643600016832352 + }, + "P2_HILout": { + "mean": 711.4987182617188, + "std": 7.856828212738037, + "lag1": -0.013758387416601181 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.4565467834472656, + "mean_abs_diff_std": 0.2373291968833655, + "mean_abs_diff_lag1": 0.4579590285817782 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9590762853622437, + "mae": 0.7737497687339783 + }, + "real_plus_synthetic": { + "rmse": 0.5409077405929565, + "mae": 0.3507160544395447 + }, + "rmse_ratio_synth_to_real": 1.7176944639028204 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8262755102040815, + "auprc": 0.5909036833874677, + "threshold": 0.004447661340236664, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6666666666666667, + "best_f1_threshold": 1.484622836112976 + }, + "real_plus_synthetic": { + "auroc": 0.8798933209647495, + "auprc": 0.646785976185459, + "threshold": 0.3300188481807709, + "f1": 0.2707423580786026, + "precision": 0.16756756756756758, + "recall": 0.7045454545454546, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.46487706899642944 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/eps_target__seed1337/eval.json b/example/results/ablations/runs/eps_target__seed1337/eval.json new file mode 100644 index 0000000..f8560b3 --- /dev/null +++ b/example/results/ablations/runs/eps_target__seed1337/eval.json @@ -0,0 +1,974 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.11384, + "std": 0.0 + }, + "P1_B2016": { + "mean": 1.2347464843750005, + "std": 0.28055916416182153 + }, + "P1_B3004": { + "mean": 396.00211, + "std": 0.0 + }, + "P1_B3005": { + "mean": 1015.43976, + "std": 0.0 + }, + "P1_B4002": { + "mean": 33.54375, + "std": 0.0 + }, + "P1_B4005": { + "mean": 32.16070861979168, + "std": 27.07097879474637 + }, + "P1_B400B": { + "mean": 1763.9296586979171, + "std": 1196.623599530601 + }, + "P1_B4022": { + "mean": 36.881583541666664, + "std": 0.14105102171131376 + }, + "P1_FCV02Z": { + "mean": 24.705335833333326, + "std": 40.29333694305376 + }, + "P1_FCV03D": { + "mean": 54.252092473958335, + "std": 9.583910714033516 + }, + "P1_FCV03Z": { + "mean": 58.00532992187499, + "std": 8.392790277486744 + }, + "P1_FT01": { + "mean": 144.8904978125, + "std": 102.31185830950737 + }, + "P1_FT01Z": { + "mean": 487.16121763020806, + "std": 466.15264685592217 + }, + "P1_FT02": { + "mean": 1129.3222571093736, + "std": 888.2477450929131 + }, + "P1_FT02Z": { + "mean": 1129.3222571093736, + "std": 888.2477450929131 + }, + "P1_FT03": { + "mean": 261.9232334114582, + "std": 57.65889509589071 + }, + "P1_FT03Z": { + "mean": 261.9232334114582, + "std": 57.65889509589071 + }, + "P1_LCV01D": { + "mean": 8.514717682291666, + "std": 6.233965307128236 + }, + "P1_LCV01Z": { + "mean": 9.904843645833335, + "std": 6.196290117428386 + }, + "P1_LIT01": { + "mean": 397.16523468750046, + "std": 25.371235226680938 + }, + "P1_PCV01D": { + "mean": 50.37298356770833, + "std": 18.727824325445198 + }, + "P1_PCV01Z": { + "mean": 56.198042916666694, + "std": 27.76267141167366 + }, + "P1_PCV02Z": { + "mean": 11.83650692708333, + "std": 0.10106959833194964 + }, + "P1_PIT01": { + "mean": 1.3783044010416647, + "std": 0.08079409440362521 + }, + "P1_PIT02": { + "mean": 0.29591864583333355, + "std": 0.0805729067204241 + }, + "P1_TIT01": { + "mean": 35.56429354166665, + "std": 0.7131287631627877 + }, + "P1_TIT02": { + "mean": 36.28562778645829, + "std": 1.423155156675057 + }, + "P2_24Vdc": { + "mean": 28.028448229166667, + "std": 0.004100860826599884 + }, + "P2_CO_rpm": { + "mean": 54097.49634807293, + "std": 30.2194448283744 + }, + "P2_HILout": { + "mean": 708.3784771354169, + "std": 16.13510484181213 + }, + "P2_MSD": { + "mean": 758.4425, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 782.5803385416674, + "std": 5.082735036026154 + }, + "P2_SIT02": { + "mean": 782.9344433333333, + "std": 4.726200835004275 + }, + "P2_VT01": { + "mean": 11.91620338541668, + "std": 0.08109007844322852 + }, + "P2_VXT02": { + "mean": -3.402698723958331, + "std": 0.6002830429870942 + }, + "P2_VXT03": { + "mean": -1.3845987239583324, + "std": 0.4172661375368494 + }, + "P2_VYT02": { + "mean": 3.8186356510416677, + "std": 0.7284637564499646 + }, + "P2_VYT03": { + "mean": 6.055064739583338, + "std": 0.687484747015177 + }, + "P3_FIT01": { + "mean": 1152.110434270833, + "std": 1788.813394402094 + }, + "P3_LCP01D": { + "mean": 598.3034625781249, + "std": 417.6954809455148 + }, + "P3_LCV01D": { + "mean": 6914.502558385419, + "std": 7046.528697187196 + }, + "P3_LIT01": { + "mean": 12507.956916796877, + "std": 5386.371328062068 + }, + "P3_PIT01": { + "mean": 591.0308262239585, + "std": 1043.4580298258338 + }, + "P4_HT_FD": { + "mean": -0.000400885416666667, + "std": 0.0027536708464229167 + }, + "P4_HT_LD": { + "mean": -115.69799739583333, + "std": 35.27327640165136 + }, + "P4_HT_PO": { + "mean": 33.98807635416668, + "std": 31.542545295262375 + }, + "P4_LD": { + "mean": 349.4122752604168, + "std": 82.57409596662433 + }, + "P4_ST_FD": { + "mean": -0.0019446614583333338, + "std": 0.010943517193000504 + }, + "P4_ST_GOV": { + "mean": 15755.961064661467, + "std": 3537.274060140035 + }, + "P4_ST_LD": { + "mean": 307.88252476562474, + "std": 64.43433701334958 + }, + "P4_ST_PO": { + "mean": 298.54694276041664, + "std": 64.61434990847361 + }, + "P4_ST_PT01": { + "mean": 10018.298350026045, + "std": 66.73776408327478 + }, + "P4_ST_TT01": { + "mean": 27591.090464270826, + "std": 44.93174774610963 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.020272347909024324, + "std_abs_err": 0.017998437082092405 + }, + "P1_B2016": { + "mean_abs_err": 0.1565198402897665, + "std_abs_err": 0.14478560168525845 + }, + "P1_B3004": { + "mean_abs_err": 7.383953824526145, + "std_abs_err": 15.41814958804181 + }, + "P1_B3005": { + "mean_abs_err": 45.609026528629784, + "std_abs_err": 61.18331651100028 + }, + "P1_B4002": { + "mean_abs_err": 1.2268236801201198, + "std_abs_err": 0.5786734991990778 + }, + "P1_B4005": { + "mean_abs_err": 28.364398396406244, + "std_abs_err": 15.110318652963318 + }, + "P1_B400B": { + "mean_abs_err": 14.546211480050943, + "std_abs_err": 3.271326611528366 + }, + "P1_B4022": { + "mean_abs_err": 0.8204088942641405, + "std_abs_err": 0.5303998331235046 + }, + "P1_FCV02Z": { + "mean_abs_err": 0.24559906713895074, + "std_abs_err": 0.1451484962044205 + }, + "P1_FCV03D": { + "mean_abs_err": 4.315770976888494, + "std_abs_err": 2.9844149953247614 + }, + "P1_FCV03Z": { + "mean_abs_err": 1.2937092583148484, + "std_abs_err": 1.6779863131551425 + }, + "P1_FT01": { + "mean_abs_err": 49.83315654378026, + "std_abs_err": 74.96387647086304 + }, + "P1_FT01Z": { + "mean_abs_err": 391.8884638739445, + "std_abs_err": 390.38694389260394 + }, + "P1_FT02": { + "mean_abs_err": 3.495896125252557, + "std_abs_err": 0.9484187584995425 + }, + "P1_FT02Z": { + "mean_abs_err": 649.1561894992751, + "std_abs_err": 305.0976106782365 + }, + "P1_FT03": { + "mean_abs_err": 20.71573686477069, + "std_abs_err": 24.18468198459521 + }, + "P1_FT03Z": { + "mean_abs_err": 799.1188049370634, + "std_abs_err": 3.4665937677266143 + }, + "P1_LCV01D": { + "mean_abs_err": 3.6396149794834702, + "std_abs_err": 3.319350354098115 + }, + "P1_LCV01Z": { + "mean_abs_err": 2.061230383504583, + "std_abs_err": 3.232820620083075 + }, + "P1_LIT01": { + "mean_abs_err": 6.949637475505824, + "std_abs_err": 9.739337178306966 + }, + "P1_PCV01D": { + "mean_abs_err": 17.092590167983047, + "std_abs_err": 5.041942375460376 + }, + "P1_PCV01Z": { + "mean_abs_err": 12.210108880235893, + "std_abs_err": 3.8980716010100167 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.0062460195215976455, + "std_abs_err": 0.004225132867302278 + }, + "P1_PIT01": { + "mean_abs_err": 0.0008231802882727735, + "std_abs_err": 0.01936153503374311 + }, + "P1_PIT02": { + "mean_abs_err": 0.19123844450818317, + "std_abs_err": 0.4181387313344557 + }, + "P1_TIT01": { + "mean_abs_err": 0.1169147007665643, + "std_abs_err": 0.12984813597683875 + }, + "P1_TIT02": { + "mean_abs_err": 0.23935089187823877, + "std_abs_err": 0.20139643065079404 + }, + "P2_24Vdc": { + "mean_abs_err": 0.0002614734610020264, + "std_abs_err": 0.0007583947121929686 + }, + "P2_CO_rpm": { + "mean_abs_err": 5.213849809748353, + "std_abs_err": 7.763713053586034 + }, + "P2_HILout": { + "mean_abs_err": 3.0843321346792436, + "std_abs_err": 7.680378964555967 + }, + "P2_MSD": { + "mean_abs_err": 4.750739999999951, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 1.3235788717755668, + "std_abs_err": 0.11618032441807102 + }, + "P2_SIT02": { + "mean_abs_err": 0.9692945492129184, + "std_abs_err": 0.46437892180869156 + }, + "P2_VT01": { + "mean_abs_err": 0.007612260593454323, + "std_abs_err": 0.012576396371226456 + }, + "P2_VXT02": { + "mean_abs_err": 0.10252025156833477, + "std_abs_err": 0.12925011671587394 + }, + "P2_VXT03": { + "mean_abs_err": 0.07770543729365453, + "std_abs_err": 0.06763875837319572 + }, + "P2_VYT02": { + "mean_abs_err": 0.16283319895126924, + "std_abs_err": 0.22855330345717678 + }, + "P2_VYT03": { + "mean_abs_err": 0.1793403732145462, + "std_abs_err": 0.30768178504408544 + }, + "P3_FIT01": { + "mean_abs_err": 1.2102581646497583, + "std_abs_err": 9.08869661181916 + }, + "P3_LCP01D": { + "mean_abs_err": 3986.4236856624493, + "std_abs_err": 4712.304516770364 + }, + "P3_LCV01D": { + "mean_abs_err": 219.37415859582234, + "std_abs_err": 206.60080137137447 + }, + "P3_LIT01": { + "mean_abs_err": 846.8875758971099, + "std_abs_err": 1032.1966911341315 + }, + "P3_PIT01": { + "mean_abs_err": 58.32707679936925, + "std_abs_err": 107.4639008813615 + }, + "P4_HT_FD": { + "mean_abs_err": 0.0002976974713149272, + "std_abs_err": 0.0007216525320188163 + }, + "P4_HT_LD": { + "mean_abs_err": 149.69291447551893, + "std_abs_err": 1.8494828916530537 + }, + "P4_HT_PO": { + "mean_abs_err": 0.0042685594097733315, + "std_abs_err": 0.10167407794089911 + }, + "P4_LD": { + "mean_abs_err": 16.716478533777547, + "std_abs_err": 21.95828959718132 + }, + "P4_ST_FD": { + "mean_abs_err": 0.001915548228450185, + "std_abs_err": 0.009301340935010475 + }, + "P4_ST_GOV": { + "mean_abs_err": 2157.0001965860574, + "std_abs_err": 1685.1308441522617 + }, + "P4_ST_LD": { + "mean_abs_err": 24.12037240419295, + "std_abs_err": 26.58116772173203 + }, + "P4_ST_PO": { + "mean_abs_err": 33.722028563242134, + "std_abs_err": 30.163406406065313 + }, + "P4_ST_PT01": { + "mean_abs_err": 32.276806793037395, + "std_abs_err": 48.95299405795565 + }, + "P4_ST_TT01": { + "mean_abs_err": 17.94222740870464, + "std_abs_err": 21.0144259415413 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 1.0, + "P1_B2016": 0.46875, + "P1_B3004": 0.8108, + "P1_B3005": 0.8106, + "P1_B4002": 0.81055, + "P1_B4005": 0.83285, + "P1_B400B": 0.7926, + "P1_B4022": 0.5730333333333334, + "P1_FCV02Z": 0.5961625, + "P1_FCV03D": 0.5442708333333334, + "P1_FCV03Z": 0.4923916666666667, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4765625, + "P1_FT02": 0.7549, + "P1_FT02Z": 0.8362499999999999, + "P1_FT03": 0.44245833333333334, + "P1_FT03Z": 1.0, + "P1_LCV01D": 0.4817708333333333, + "P1_LCV01Z": 0.4546666666666667, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5577125, + "P1_PCV01Z": 0.5402958333333334, + "P1_PCV02Z": 0.765625, + "P1_PIT01": 0.14325833333333338, + "P1_PIT02": 0.25495, + "P1_TIT01": 0.3229166666666667, + "P1_TIT02": 0.4635416666666667, + "P2_24Vdc": 0.12129166666666658, + "P2_CO_rpm": 0.23795, + "P2_HILout": 0.08199583333333332, + "P2_MSD": 1.0, + "P2_SIT01": 0.498375, + "P2_SIT02": 0.5013666666666667, + "P2_VT01": 0.12296666666666667, + "P2_VXT02": 0.2818291666666666, + "P2_VXT03": 0.43662083333333335, + "P2_VYT02": 0.25022500000000003, + "P2_VYT03": 0.2700083333333334, + "P3_FIT01": 0.4427083333333333, + "P3_LCP01D": 0.56185, + "P3_LCV01D": 0.4427083333333333, + "P3_LIT01": 0.3020833333333333, + "P3_PIT01": 0.1158541666666667, + "P4_HT_FD": 0.09770000000000001, + "P4_HT_LD": 0.9375, + "P4_HT_PO": 0.19570416666666668, + "P4_LD": 0.2786458333333333, + "P4_ST_FD": 0.041666666666666664, + "P4_ST_GOV": 0.5416666666666666, + "P4_ST_LD": 0.375, + "P4_ST_PO": 0.4739583333333333, + "P4_ST_PT01": 0.2760416666666667, + "P4_ST_TT01": 0.47985 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.08333, + "q25_diff": 0.012630000000000002, + "q50_diff": 0.012630000000000002, + "q75_diff": 0.012630000000000002, + "q95_diff": 0.012630000000000002 + }, + "P1_B2016": { + "q05_diff": 0.20477000000000012, + "q25_diff": 0.35475999999999996, + "q50_diff": 0.0053399999999999, + "q75_diff": 0.014140000000000041, + "q95_diff": 0.030200000000000005 + }, + "P1_B3004": { + "q05_diff": 2.8398500000000126, + "q25_diff": 1.6357399999999984, + "q50_diff": 1.6357399999999984, + "q75_diff": 1.6357399999999984, + "q95_diff": 1.6357399999999984 + }, + "P1_B3005": { + "q05_diff": 13.44177000000002, + "q25_diff": 13.44177000000002, + "q50_diff": 13.44177000000002, + "q75_diff": 13.44177000000002, + "q95_diff": 99.67999000000009 + }, + "P1_B4002": { + "q05_diff": 1.5437500000000028, + "q25_diff": 0.11175000000000068, + "q50_diff": 0.11175000000000068, + "q75_diff": 0.11175000000000068, + "q95_diff": 0.11175000000000068 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 100.0, + "q50_diff": 45.09439, + "q75_diff": 45.09439, + "q95_diff": 45.09439 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2479.86694, + "q50_diff": 192.11987999999974, + "q75_diff": 14.038080000000264, + "q95_diff": 20.074460000000272 + }, + "P1_B4022": { + "q05_diff": 1.2294699999999992, + "q25_diff": 0.1574500000000043, + "q50_diff": 0.4875299999999996, + "q75_diff": 0.5946499999999943, + "q95_diff": 0.6461800000000011 + }, + "P1_FCV02Z": { + "q05_diff": 0.015249999999999986, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.07630000000000003, + "q75_diff": 41.984559999999995, + "q95_diff": 0.026700000000005275 + }, + "P1_FCV03D": { + "q05_diff": 5.226129999999998, + "q25_diff": 5.662500000000001, + "q50_diff": 5.984720000000003, + "q75_diff": 11.15401, + "q95_diff": 0.773469999999989 + }, + "P1_FCV03Z": { + "q05_diff": 5.432130000000001, + "q25_diff": 5.790710000000004, + "q50_diff": 8.96454, + "q75_diff": 11.566150000000007, + "q95_diff": 0.5950900000000132 + }, + "P1_FT01": { + "q05_diff": 166.51155, + "q25_diff": 181.00736999999998, + "q50_diff": 32.806329999999974, + "q75_diff": 37.76549, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 783.07349, + "q25_diff": 816.12396, + "q50_diff": 74.79840000000002, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.20669, + "q50_diff": 260.41661999999997, + "q75_diff": 15.831050000000005, + "q95_diff": 24.032470000000103 + }, + "P1_FT02Z": { + "q05_diff": 27.82217, + "q25_diff": 2761.14748, + "q50_diff": 1119.9909699999998, + "q75_diff": 873.6931100000002, + "q95_diff": 879.7294900000002 + }, + "P1_FT03": { + "q05_diff": 57.95555999999999, + "q25_diff": 59.09995999999998, + "q50_diff": 51.52633000000003, + "q75_diff": 59.755690000000016, + "q95_diff": 1.9073100000000522 + }, + "P1_FT03Z": { + "q05_diff": 810.6894199999999, + "q25_diff": 813.2986699999999, + "q50_diff": 703.89301, + "q75_diff": 697.61667, + "q95_diff": 800.20041 + }, + "P1_LCV01D": { + "q05_diff": 5.01824, + "q25_diff": 5.7707999999999995, + "q50_diff": 4.63551, + "q75_diff": 3.31104, + "q95_diff": 0.6187400000000007 + }, + "P1_LCV01Z": { + "q05_diff": 6.52313, + "q25_diff": 7.3547400000000005, + "q50_diff": 4.9746500000000005, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.710700000000001 + }, + "P1_LIT01": { + "q05_diff": 35.884119999999996, + "q25_diff": 1.0223700000000235, + "q50_diff": 6.267770000000041, + "q75_diff": 8.127619999999979, + "q95_diff": 34.6062 + }, + "P1_PCV01D": { + "q05_diff": 11.05535, + "q25_diff": 15.505350000000004, + "q50_diff": 22.272359999999992, + "q75_diff": 18.644169999999995, + "q95_diff": 12.081639999999993 + }, + "P1_PCV01Z": { + "q05_diff": 10.925290000000004, + "q25_diff": 15.548700000000004, + "q50_diff": 22.53723, + "q75_diff": 36.48376, + "q95_diff": 30.075069999999997 + }, + "P1_PCV02Z": { + "q05_diff": 0.2441399999999998, + "q25_diff": 0.2441399999999998, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.22125000000000128, + "q95_diff": 0.0 + }, + "P1_PIT01": { + "q05_diff": 0.034660000000000135, + "q25_diff": 0.017550000000000177, + "q50_diff": 0.0045800000000000285, + "q75_diff": 0.021360000000000046, + "q95_diff": 0.008389999999999898 + }, + "P1_PIT02": { + "q05_diff": 0.0121, + "q25_diff": 0.0015199999999999658, + "q50_diff": 0.0007699999999999929, + "q75_diff": 0.04196, + "q95_diff": 0.14801000000000003 + }, + "P1_TIT01": { + "q05_diff": 0.0915600000000012, + "q25_diff": 0.44250000000000256, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.2428200000000018, + "q95_diff": 0.01525999999999783 + }, + "P1_TIT02": { + "q05_diff": 0.10681000000000296, + "q25_diff": 0.4730199999999982, + "q50_diff": 0.06103999999999843, + "q75_diff": 0.2916799999999995, + "q95_diff": 0.4425099999999986 + }, + "P2_24Vdc": { + "q05_diff": 0.0008400000000001739, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.0008799999999986596, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.000920000000000698 + }, + "P2_CO_rpm": { + "q05_diff": 11.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 2.0, + "q95_diff": 17.0 + }, + "P2_HILout": { + "q05_diff": 44.20165999999995, + "q25_diff": 2.160639999999944, + "q50_diff": 0.08429000000000997, + "q75_diff": 1.6235400000000482, + "q95_diff": 2.6306200000000217 + }, + "P2_MSD": { + "q05_diff": 4.750739999999951, + "q25_diff": 4.750739999999951, + "q50_diff": 4.750739999999951, + "q75_diff": 4.750739999999951, + "q95_diff": 4.750739999999951 + }, + "P2_SIT01": { + "q05_diff": 3.0, + "q25_diff": 3.0, + "q50_diff": 6.0, + "q75_diff": 5.0, + "q95_diff": 1.0 + }, + "P2_SIT02": { + "q05_diff": 2.6183500000000777, + "q25_diff": 3.569460000000049, + "q50_diff": 5.619690000000105, + "q75_diff": 5.63000999999997, + "q95_diff": 1.9693599999999378 + }, + "P2_VT01": { + "q05_diff": 0.05377999999999972, + "q25_diff": 0.01501000000000019, + "q50_diff": 0.014210000000000278, + "q75_diff": 0.01332000000000022, + "q95_diff": 0.018940000000000623 + }, + "P2_VXT02": { + "q05_diff": 0.23179999999999978, + "q25_diff": 0.2963999999999998, + "q50_diff": 0.2848099999999998, + "q75_diff": 0.26656000000000013, + "q95_diff": 0.4363999999999999 + }, + "P2_VXT03": { + "q05_diff": 0.2124999999999999, + "q25_diff": 0.26927999999999996, + "q50_diff": 0.29759, + "q75_diff": 0.3269500000000001, + "q95_diff": 0.49360000000000004 + }, + "P2_VYT02": { + "q05_diff": 0.7055000000000002, + "q25_diff": 0.262, + "q50_diff": 0.2273900000000002, + "q75_diff": 0.23532999999999937, + "q95_diff": 0.2586000000000004 + }, + "P2_VYT03": { + "q05_diff": 1.1566, + "q25_diff": 0.1858500000000003, + "q50_diff": 0.1757800000000005, + "q75_diff": 0.18257000000000012, + "q95_diff": 0.23179999999999978 + }, + "P3_FIT01": { + "q05_diff": 5.0, + "q25_diff": 7.0, + "q50_diff": 48.98575, + "q75_diff": 82.06982000000016, + "q95_diff": 15.599610000000212 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 816.0, + "q75_diff": 8768.0, + "q95_diff": 12664.0 + }, + "P3_LCV01D": { + "q05_diff": 16.0, + "q25_diff": 336.0, + "q50_diff": 3223.17822, + "q75_diff": 171.33397999999943, + "q95_diff": 32.0 + }, + "P3_LIT01": { + "q05_diff": 1399.0, + "q25_diff": 4721.0, + "q50_diff": 639.3535200000006, + "q75_diff": 1142.2949200000003, + "q95_diff": 17.0 + }, + "P3_PIT01": { + "q05_diff": 1.0, + "q25_diff": 0.0, + "q50_diff": 1.0, + "q75_diff": 206.0, + "q95_diff": 346.0 + }, + "P4_HT_FD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0005099999999999999 + }, + "P4_HT_LD": { + "q05_diff": 124.7862, + "q25_diff": 124.7862, + "q50_diff": 179.53157, + "q75_diff": 193.36982, + "q95_diff": 56.57957999999999 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 1.35638, + "q50_diff": 14.677540000000004, + "q75_diff": 0.8499099999999942, + "q95_diff": 0.10007000000000232 + }, + "P4_LD": { + "q05_diff": 38.17633000000001, + "q25_diff": 89.59051, + "q50_diff": 3.428100000000029, + "q75_diff": 0.13958000000002357, + "q95_diff": 4.319270000000017 + }, + "P4_ST_FD": { + "q05_diff": 2.9999999999999645e-05, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 8.000000000000021e-05 + }, + "P4_ST_GOV": { + "q05_diff": 2299.0, + "q25_diff": 4178.0, + "q50_diff": 5178.0, + "q75_diff": 106.94727000000057, + "q95_diff": 415.2753899999989 + }, + "P4_ST_LD": { + "q05_diff": 39.333740000000034, + "q25_diff": 76.64203000000003, + "q50_diff": 0.368720000000053, + "q75_diff": 2.007509999999968, + "q95_diff": 7.758240000000001 + }, + "P4_ST_PO": { + "q05_diff": 43.02301, + "q25_diff": 78.10687000000001, + "q50_diff": 0.6149300000000153, + "q75_diff": 2.1597300000000246, + "q95_diff": 7.8603200000000015 + }, + "P4_ST_PT01": { + "q05_diff": 108.0, + "q25_diff": 116.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P4_ST_TT01": { + "q05_diff": 22.0, + "q25_diff": 56.0, + "q50_diff": 1.2617199999986042, + "q75_diff": 1.0, + "q95_diff": 1.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0189476811306442e-05, + "P1_B2016": 0.8239161706376917, + "P1_B3004": 1.018740504876714e-05, + "P1_B3005": 0.9999898125942444, + "P1_B4002": 0.9999898146591784, + "P1_B4005": 1.045455737564053, + "P1_B400B": 1.000823178812837, + "P1_B4022": 0.03037469907643997, + "P1_FCV02Z": 0.8990099474564699, + "P1_FCV03D": 1.0583750038510076, + "P1_FCV03Z": 0.942693336908545, + "P1_FT01": 0.9097352918666402, + "P1_FT01Z": 0.912762944961449, + "P1_FT02": 0.9117761892474533, + "P1_FT02Z": 0.9118311678573648, + "P1_FT03": 0.9305900408187426, + "P1_FT03Z": 0.9304529837741358, + "P1_LCV01D": 1.0498654038050619, + "P1_LCV01Z": 0.883817319747234, + "P1_LIT01": 0.8559220185741317, + "P1_PCV01D": 0.8177877924596789, + "P1_PCV01Z": 1.044134778199803, + "P1_PCV02Z": 0.5798489812635991, + "P1_PIT01": 0.9302862520506019, + "P1_PIT02": 0.8133964033423414, + "P1_TIT01": 0.8348426617871705, + "P1_TIT02": 0.950787093191726, + "P2_24Vdc": 0.015362247302245204, + "P2_CO_rpm": 0.4019486663200069, + "P2_HILout": 0.007318973791744204, + "P2_MSD": 0.0, + "P2_SIT01": 0.49156029781639277, + "P2_SIT02": 0.763964286715772, + "P2_VT01": 0.3279489450854321, + "P2_VXT02": 0.6829469029404855, + "P2_VXT03": 0.7257564665587917, + "P2_VYT02": 0.708956109529326, + "P2_VYT03": 0.6451582481202596, + "P3_FIT01": 0.7874574045605387, + "P3_LCP01D": 1.0276466561226463, + "P3_LCV01D": 0.9954988909036026, + "P3_LIT01": 0.960072556685037, + "P3_PIT01": 0.8883587815956718, + "P4_HT_FD": 0.26400019639364447, + "P4_HT_LD": 0.15670838558697175, + "P4_HT_PO": 0.9415104363827507, + "P4_LD": 0.9696178375756026, + "P4_ST_FD": 0.28595359004958487, + "P4_ST_GOV": 0.9454050192942649, + "P4_ST_LD": 0.9343184982665232, + "P4_ST_PO": 0.927849947228268, + "P4_ST_PT01": 0.8413419488661078, + "P4_ST_TT01": 0.8017488548689526 + }, + "discrete_jsd": { + "P1_FCV01D": 0.7708162008591284, + "P1_FCV01Z": 0.9973184330893802, + "P1_FCV02D": 0.7241545324106555, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.04436084816766643, + "P4_ST_PS": 0.12530984363905556 + }, + "avg_ks": 0.48224064465408806, + "avg_jsd": 0.10238307146791868, + "avg_lag1_diff": 0.7276772745273601 +} \ No newline at end of file diff --git a/example/results/ablations/runs/eps_target__seed1337/eval_post.json b/example/results/ablations/runs/eps_target__seed1337/eval_post.json new file mode 100644 index 0000000..f9563c7 --- /dev/null +++ b/example/results/ablations/runs/eps_target__seed1337/eval_post.json @@ -0,0 +1,971 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09355486979166669, + "std": 0.01799194293084728 + }, + "P1_B2016": { + "mean": 1.2347464843750005, + "std": 0.28055916416182153 + }, + "P1_B3004": { + "mean": 403.2905014583335, + "std": 14.217563527951576 + }, + "P1_B3005": { + "mean": 1059.5253898958326, + "std": 58.42587703751758 + }, + "P1_B4002": { + "mean": 32.37490617187499, + "std": 0.6519533213854061 + }, + "P1_B4005": { + "mean": 64.99571291666655, + "std": 42.03467331899641 + }, + "P1_B400B": { + "mean": 1763.9296586979171, + "std": 1196.623599530601 + }, + "P1_B4022": { + "mean": 36.08671476562498, + "std": 0.7147782383668128 + }, + "P1_FCV02Z": { + "mean": 21.954832630208315, + "std": 38.87335152465215 + }, + "P1_FCV03D": { + "mean": 58.14541601562505, + "std": 6.267267768996455 + }, + "P1_FCV03Z": { + "mean": 58.21398343750003, + "std": 6.423211358701469 + }, + "P1_FT01": { + "mean": 144.8904978125, + "std": 102.31185830950737 + }, + "P1_FT01Z": { + "mean": 487.16121763020806, + "std": 466.15264685592217 + }, + "P1_FT02": { + "mean": 1129.3222571093736, + "std": 888.2477450929131 + }, + "P1_FT02Z": { + "mean": 1843.7261292187509, + "std": 1190.4241853713818 + }, + "P1_FT03": { + "mean": 261.9232334114582, + "std": 57.65889509589071 + }, + "P1_FT03Z": { + "mean": 1058.2564329947922, + "std": 59.50957255113764 + }, + "P1_LCV01D": { + "mean": 11.699055963541667, + "std": 3.0155193220433567 + }, + "P1_LCV01Z": { + "mean": 11.627510442708333, + "std": 3.0251072332459787 + }, + "P1_LIT01": { + "mean": 397.16523468750046, + "std": 25.371235226680938 + }, + "P1_PCV01D": { + "mean": 65.64593611979173, + "std": 23.56629129628467 + }, + "P1_PCV01Z": { + "mean": 65.565628671875, + "std": 23.790956943468952 + }, + "P1_PCV02Z": { + "mean": 11.867279765625007, + "std": 0.10811550492611105 + }, + "P1_PIT01": { + "mean": 1.3783044010416647, + "std": 0.08079409440362521 + }, + "P1_PIT02": { + "mean": 0.29591864583333355, + "std": 0.0805729067204241 + }, + "P1_TIT01": { + "mean": 35.56429354166665, + "std": 0.7131287631627877 + }, + "P1_TIT02": { + "mean": 36.28562778645829, + "std": 1.423155156675057 + }, + "P2_24Vdc": { + "mean": 28.02860809895833, + "std": 0.0032970984602375326 + }, + "P2_CO_rpm": { + "mean": 54097.49634807293, + "std": 30.2194448283744 + }, + "P2_HILout": { + "mean": 711.4986896354167, + "std": 7.856828438379631 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 782.5803385416674, + "std": 5.082735036026154 + }, + "P2_SIT02": { + "mean": 782.9344433333333, + "std": 4.726200835004275 + }, + "P2_VT01": { + "mean": 11.91620338541668, + "std": 0.08109007844322852 + }, + "P2_VXT02": { + "mean": -3.402698723958331, + "std": 0.6002830429870942 + }, + "P2_VXT03": { + "mean": -1.3845987239583324, + "std": 0.4172661375368494 + }, + "P2_VYT02": { + "mean": 3.8186356510416677, + "std": 0.7284637564499646 + }, + "P2_VYT03": { + "mean": 6.055064739583338, + "std": 0.687484747015177 + }, + "P3_FIT01": { + "mean": 1152.110434270833, + "std": 1788.813394402094 + }, + "P3_LCP01D": { + "mean": 598.3034625781249, + "std": 417.6954809455148 + }, + "P3_LCV01D": { + "mean": 6914.502558385419, + "std": 7046.528697187196 + }, + "P3_LIT01": { + "mean": 12507.956916796877, + "std": 5386.371328062068 + }, + "P3_PIT01": { + "mean": 591.0308262239585, + "std": 1043.4580298258338 + }, + "P4_HT_FD": { + "mean": -0.000400885416666667, + "std": 0.0027536708464229167 + }, + "P4_HT_LD": { + "mean": 34.231568880208286, + "std": 33.60361854009684 + }, + "P4_HT_PO": { + "mean": 35.25451283854166, + "std": 31.650898099038645 + }, + "P4_LD": { + "mean": 349.4122752604168, + "std": 82.57409596662433 + }, + "P4_ST_FD": { + "mean": -0.0019446614583333338, + "std": 0.010943517193000504 + }, + "P4_ST_GOV": { + "mean": 15755.961064661467, + "std": 3537.274060140035 + }, + "P4_ST_LD": { + "mean": 307.88252476562474, + "std": 64.43433701334958 + }, + "P4_ST_PO": { + "mean": 298.54694276041664, + "std": 64.61434990847361 + }, + "P4_ST_PT01": { + "mean": 10018.298350026045, + "std": 66.73776408327478 + }, + "P4_ST_TT01": { + "mean": 27591.090464270826, + "std": 44.93174774610963 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 1.2782299308983225e-05, + "std_abs_err": 6.494151245123314e-06 + }, + "P1_B2016": { + "mean_abs_err": 0.1565198402897665, + "std_abs_err": 0.14478560168525845 + }, + "P1_B3004": { + "mean_abs_err": 0.09556236619266656, + "std_abs_err": 1.2005860600902345 + }, + "P1_B3005": { + "mean_abs_err": 1.5233966327971302, + "std_abs_err": 2.757439473482698 + }, + "P1_B4002": { + "mean_abs_err": 0.05797985199510691, + "std_abs_err": 0.07327982218632823 + }, + "P1_B4005": { + "mean_abs_err": 4.470605900468627, + "std_abs_err": 0.14662412871327746 + }, + "P1_B400B": { + "mean_abs_err": 14.546211480050943, + "std_abs_err": 3.271326611528366 + }, + "P1_B4022": { + "mean_abs_err": 0.02554011822245883, + "std_abs_err": 0.04332738353199439 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.996102270263961, + "std_abs_err": 1.5651339146060295 + }, + "P1_FCV03D": { + "mean_abs_err": 0.42244743522177686, + "std_abs_err": 0.3322279497122995 + }, + "P1_FCV03Z": { + "mean_abs_err": 1.0850557426898035, + "std_abs_err": 0.29159260563013323 + }, + "P1_FT01": { + "mean_abs_err": 49.83315654378026, + "std_abs_err": 74.96387647086304 + }, + "P1_FT01Z": { + "mean_abs_err": 391.8884638739445, + "std_abs_err": 390.38694389260394 + }, + "P1_FT02": { + "mean_abs_err": 3.495896125252557, + "std_abs_err": 0.9484187584995425 + }, + "P1_FT02Z": { + "mean_abs_err": 65.24768261010217, + "std_abs_err": 2.9211703997677887 + }, + "P1_FT03": { + "mean_abs_err": 20.71573686477069, + "std_abs_err": 24.18468198459521 + }, + "P1_FT03Z": { + "mean_abs_err": 2.785605353729352, + "std_abs_err": 1.6159163124796905 + }, + "P1_LCV01D": { + "mean_abs_err": 0.45527669823346884, + "std_abs_err": 0.1009043690132354 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.33856358662958463, + "std_abs_err": 0.061637735900667234 + }, + "P1_LIT01": { + "mean_abs_err": 6.949637475505824, + "std_abs_err": 9.739337178306966 + }, + "P1_PCV01D": { + "mean_abs_err": 1.8196376158996515, + "std_abs_err": 0.20347540462090308 + }, + "P1_PCV01Z": { + "mean_abs_err": 2.842523125027583, + "std_abs_err": 0.07364286719469249 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.024526819020080026, + "std_abs_err": 0.011271039461463683 + }, + "P1_PIT01": { + "mean_abs_err": 0.0008231802882727735, + "std_abs_err": 0.01936153503374311 + }, + "P1_PIT02": { + "mean_abs_err": 0.19123844450818317, + "std_abs_err": 0.4181387313344557 + }, + "P1_TIT01": { + "mean_abs_err": 0.1169147007665643, + "std_abs_err": 0.12984813597683875 + }, + "P1_TIT02": { + "mean_abs_err": 0.23935089187823877, + "std_abs_err": 0.20139643065079404 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00010160366933931186, + "std_abs_err": 4.536765416938244e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 5.213849809748353, + "std_abs_err": 7.763713053586034 + }, + "P2_HILout": { + "mean_abs_err": 0.035880365320508645, + "std_abs_err": 0.5978974388765312 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 1.3235788717755668, + "std_abs_err": 0.11618032441807102 + }, + "P2_SIT02": { + "mean_abs_err": 0.9692945492129184, + "std_abs_err": 0.46437892180869156 + }, + "P2_VT01": { + "mean_abs_err": 0.007612260593454323, + "std_abs_err": 0.012576396371226456 + }, + "P2_VXT02": { + "mean_abs_err": 0.10252025156833477, + "std_abs_err": 0.12925011671587394 + }, + "P2_VXT03": { + "mean_abs_err": 0.07770543729365453, + "std_abs_err": 0.06763875837319572 + }, + "P2_VYT02": { + "mean_abs_err": 0.16283319895126924, + "std_abs_err": 0.22855330345717678 + }, + "P2_VYT03": { + "mean_abs_err": 0.1793403732145462, + "std_abs_err": 0.30768178504408544 + }, + "P3_FIT01": { + "mean_abs_err": 1.2102581646497583, + "std_abs_err": 9.08869661181916 + }, + "P3_LCP01D": { + "mean_abs_err": 3986.4236856624493, + "std_abs_err": 4712.304516770364 + }, + "P3_LCV01D": { + "mean_abs_err": 219.37415859582234, + "std_abs_err": 206.60080137137447 + }, + "P3_LIT01": { + "mean_abs_err": 846.8875758971099, + "std_abs_err": 1032.1966911341315 + }, + "P3_PIT01": { + "mean_abs_err": 58.32707679936925, + "std_abs_err": 107.4639008813615 + }, + "P4_HT_FD": { + "mean_abs_err": 0.0002976974713149272, + "std_abs_err": 0.0007216525320188163 + }, + "P4_HT_LD": { + "mean_abs_err": 0.23665180052267232, + "std_abs_err": 0.17982503009853446 + }, + "P4_HT_PO": { + "mean_abs_err": 1.2707050437847514, + "std_abs_err": 0.21002688171716954 + }, + "P4_LD": { + "mean_abs_err": 16.716478533777547, + "std_abs_err": 21.95828959718132 + }, + "P4_ST_FD": { + "mean_abs_err": 0.001915548228450185, + "std_abs_err": 0.009301340935010475 + }, + "P4_ST_GOV": { + "mean_abs_err": 2157.0001965860574, + "std_abs_err": 1685.1308441522617 + }, + "P4_ST_LD": { + "mean_abs_err": 24.12037240419295, + "std_abs_err": 26.58116772173203 + }, + "P4_ST_PO": { + "mean_abs_err": 33.722028563242134, + "std_abs_err": 30.163406406065313 + }, + "P4_ST_PT01": { + "mean_abs_err": 32.276806793037395, + "std_abs_err": 48.95299405795565 + }, + "P4_ST_TT01": { + "mean_abs_err": 17.94222740870464, + "std_abs_err": 21.0144259415413 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.6595583333333334, + "P1_B2016": 0.46875, + "P1_B3004": 0.671875, + "P1_B3005": 0.5293, + "P1_B4002": 0.7923208333333334, + "P1_B4005": 0.3062625, + "P1_B400B": 0.7926, + "P1_B4022": 0.5756333333333334, + "P1_FCV02Z": 0.7746, + "P1_FCV03D": 0.45162500000000005, + "P1_FCV03Z": 0.40985833333333327, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4765625, + "P1_FT02": 0.7549, + "P1_FT02Z": 0.4029958333333334, + "P1_FT03": 0.44245833333333334, + "P1_FT03Z": 0.46950833333333336, + "P1_LCV01D": 0.3859916666666667, + "P1_LCV01Z": 0.397625, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5304583333333333, + "P1_PCV01Z": 0.5184416666666667, + "P1_PCV02Z": 0.6588541666666666, + "P1_PIT01": 0.14325833333333338, + "P1_PIT02": 0.25495, + "P1_TIT01": 0.3229166666666667, + "P1_TIT02": 0.4635416666666667, + "P2_24Vdc": 0.13548333333333334, + "P2_CO_rpm": 0.23795, + "P2_HILout": 0.08242499999999997, + "P2_MSD": 0.0, + "P2_SIT01": 0.498375, + "P2_SIT02": 0.5013666666666667, + "P2_VT01": 0.12296666666666667, + "P2_VXT02": 0.2818291666666666, + "P2_VXT03": 0.43662083333333335, + "P2_VYT02": 0.25022500000000003, + "P2_VYT03": 0.2700083333333334, + "P3_FIT01": 0.4427083333333333, + "P3_LCP01D": 0.56185, + "P3_LCV01D": 0.4427083333333333, + "P3_LIT01": 0.3020833333333333, + "P3_PIT01": 0.1158541666666667, + "P4_HT_FD": 0.09770000000000001, + "P4_HT_LD": 0.06742500000000001, + "P4_HT_PO": 0.05970416666666667, + "P4_LD": 0.2786458333333333, + "P4_ST_FD": 0.041666666666666664, + "P4_ST_GOV": 0.5416666666666666, + "P4_ST_LD": 0.375, + "P4_ST_PO": 0.4739583333333333, + "P4_ST_PT01": 0.2760416666666667, + "P4_ST_TT01": 0.47985 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.02845, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0011099999999999999, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.20477000000000012, + "q25_diff": 0.35475999999999996, + "q50_diff": 0.0053399999999999, + "q75_diff": 0.014140000000000041, + "q95_diff": 0.030200000000000005 + }, + "P1_B3004": { + "q05_diff": 14.80313000000001, + "q25_diff": 1.6197799999999916, + "q50_diff": 6.726189999999974, + "q75_diff": 8.508819999999957, + "q95_diff": 37.86444 + }, + "P1_B3005": { + "q05_diff": 40.40789999999993, + "q25_diff": 8.045049999999947, + "q50_diff": 89.51202, + "q75_diff": 104.97393, + "q95_diff": 4.4593499999998585 + }, + "P1_B4002": { + "q05_diff": 0.4564699999999995, + "q25_diff": 1.6555000000000035, + "q50_diff": 1.6555000000000035, + "q75_diff": 0.9287000000000063, + "q95_diff": 0.16969000000000278 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 81.96193, + "q50_diff": 0.04000000000000625, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2479.86694, + "q50_diff": 192.11987999999974, + "q75_diff": 14.038080000000264, + "q95_diff": 20.074460000000272 + }, + "P1_B4022": { + "q05_diff": 0.38222999999999985, + "q25_diff": 1.3479000000000028, + "q50_diff": 1.4273799999999994, + "q75_diff": 0.9127399999999994, + "q95_diff": 0.5270299999999963 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.05340999999999996, + "q75_diff": 38.169869999999996, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 1.3654200000000003, + "q25_diff": 0.2554099999999977, + "q50_diff": 8.60333, + "q75_diff": 10.84863, + "q95_diff": 0.824300000000008 + }, + "P1_FCV03Z": { + "q05_diff": 1.7547599999999974, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.148199999999996, + "q75_diff": 10.864249999999998, + "q95_diff": 0.9307899999999876 + }, + "P1_FT01": { + "q05_diff": 166.51155, + "q25_diff": 181.00736999999998, + "q50_diff": 32.806329999999974, + "q75_diff": 37.76549, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 783.07349, + "q25_diff": 816.12396, + "q50_diff": 74.79840000000002, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.20669, + "q50_diff": 260.41661999999997, + "q75_diff": 15.831050000000005, + "q95_diff": 24.032470000000103 + }, + "P1_FT02Z": { + "q05_diff": 1.9862300000000026, + "q25_diff": 2300.54303, + "q50_diff": 14.318849999999657, + "q75_diff": 7.299800000000232, + "q95_diff": 6.317140000000109 + }, + "P1_FT03": { + "q05_diff": 57.95555999999999, + "q25_diff": 59.09995999999998, + "q50_diff": 51.52633000000003, + "q75_diff": 59.755690000000016, + "q95_diff": 1.9073100000000522 + }, + "P1_FT03Z": { + "q05_diff": 23.41980000000001, + "q25_diff": 2.9805299999999306, + "q50_diff": 84.98266000000012, + "q75_diff": 100.6383699999999, + "q95_diff": 3.5079399999999623 + }, + "P1_LCV01D": { + "q05_diff": 0.17497000000000007, + "q25_diff": 0.29953000000000074, + "q50_diff": 4.171619999999999, + "q75_diff": 3.162939999999999, + "q95_diff": 0.7125800000000009 + }, + "P1_LCV01Z": { + "q05_diff": 0.0, + "q25_diff": 0.5874600000000001, + "q50_diff": 4.638670000000001, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.9536700000000007 + }, + "P1_LIT01": { + "q05_diff": 35.884119999999996, + "q25_diff": 1.0223700000000235, + "q50_diff": 6.267770000000041, + "q75_diff": 8.127619999999979, + "q95_diff": 34.6062 + }, + "P1_PCV01D": { + "q05_diff": 0.18625000000000114, + "q25_diff": 2.7490099999999984, + "q50_diff": 18.274499999999996, + "q75_diff": 43.266600000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.22888000000000375, + "q25_diff": 2.0370500000000007, + "q50_diff": 16.029360000000004, + "q75_diff": 43.33495, + "q95_diff": 45.1493 + }, + "P1_PCV02Z": { + "q05_diff": 0.2288800000000002, + "q25_diff": 0.2212499999999995, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_PIT01": { + "q05_diff": 0.034660000000000135, + "q25_diff": 0.017550000000000177, + "q50_diff": 0.0045800000000000285, + "q75_diff": 0.021360000000000046, + "q95_diff": 0.008389999999999898 + }, + "P1_PIT02": { + "q05_diff": 0.0121, + "q25_diff": 0.0015199999999999658, + "q50_diff": 0.0007699999999999929, + "q75_diff": 0.04196, + "q95_diff": 0.14801000000000003 + }, + "P1_TIT01": { + "q05_diff": 0.0915600000000012, + "q25_diff": 0.44250000000000256, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.2428200000000018, + "q95_diff": 0.01525999999999783 + }, + "P1_TIT02": { + "q05_diff": 0.10681000000000296, + "q25_diff": 0.4730199999999982, + "q50_diff": 0.06103999999999843, + "q75_diff": 0.2916799999999995, + "q95_diff": 0.4425099999999986 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.00013999999999825263, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.00010999999999938836 + }, + "P2_CO_rpm": { + "q05_diff": 11.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 2.0, + "q95_diff": 17.0 + }, + "P2_HILout": { + "q05_diff": 0.3295900000000529, + "q25_diff": 1.6540499999999838, + "q50_diff": 0.31737999999995736, + "q75_diff": 1.7334000000000742, + "q95_diff": 2.2399900000000343 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 3.0, + "q25_diff": 3.0, + "q50_diff": 6.0, + "q75_diff": 5.0, + "q95_diff": 1.0 + }, + "P2_SIT02": { + "q05_diff": 2.6183500000000777, + "q25_diff": 3.569460000000049, + "q50_diff": 5.619690000000105, + "q75_diff": 5.63000999999997, + "q95_diff": 1.9693599999999378 + }, + "P2_VT01": { + "q05_diff": 0.05377999999999972, + "q25_diff": 0.01501000000000019, + "q50_diff": 0.014210000000000278, + "q75_diff": 0.01332000000000022, + "q95_diff": 0.018940000000000623 + }, + "P2_VXT02": { + "q05_diff": 0.23179999999999978, + "q25_diff": 0.2963999999999998, + "q50_diff": 0.2848099999999998, + "q75_diff": 0.26656000000000013, + "q95_diff": 0.4363999999999999 + }, + "P2_VXT03": { + "q05_diff": 0.2124999999999999, + "q25_diff": 0.26927999999999996, + "q50_diff": 0.29759, + "q75_diff": 0.3269500000000001, + "q95_diff": 0.49360000000000004 + }, + "P2_VYT02": { + "q05_diff": 0.7055000000000002, + "q25_diff": 0.262, + "q50_diff": 0.2273900000000002, + "q75_diff": 0.23532999999999937, + "q95_diff": 0.2586000000000004 + }, + "P2_VYT03": { + "q05_diff": 1.1566, + "q25_diff": 0.1858500000000003, + "q50_diff": 0.1757800000000005, + "q75_diff": 0.18257000000000012, + "q95_diff": 0.23179999999999978 + }, + "P3_FIT01": { + "q05_diff": 5.0, + "q25_diff": 7.0, + "q50_diff": 48.98575, + "q75_diff": 82.06982000000016, + "q95_diff": 15.599610000000212 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 816.0, + "q75_diff": 8768.0, + "q95_diff": 12664.0 + }, + "P3_LCV01D": { + "q05_diff": 16.0, + "q25_diff": 336.0, + "q50_diff": 3223.17822, + "q75_diff": 171.33397999999943, + "q95_diff": 32.0 + }, + "P3_LIT01": { + "q05_diff": 1399.0, + "q25_diff": 4721.0, + "q50_diff": 639.3535200000006, + "q75_diff": 1142.2949200000003, + "q95_diff": 17.0 + }, + "P3_PIT01": { + "q05_diff": 1.0, + "q25_diff": 0.0, + "q50_diff": 1.0, + "q75_diff": 206.0, + "q95_diff": 346.0 + }, + "P4_HT_FD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0005099999999999999 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 27.14122, + "q75_diff": 0.13020999999999106, + "q95_diff": 0.21701000000000192 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 0.9223299999999999, + "q50_diff": 8.89752, + "q75_diff": 0.054199999999994475, + "q95_diff": 0.6510700000000043 + }, + "P4_LD": { + "q05_diff": 38.17633000000001, + "q25_diff": 89.59051, + "q50_diff": 3.428100000000029, + "q75_diff": 0.13958000000002357, + "q95_diff": 4.319270000000017 + }, + "P4_ST_FD": { + "q05_diff": 2.9999999999999645e-05, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 8.000000000000021e-05 + }, + "P4_ST_GOV": { + "q05_diff": 2299.0, + "q25_diff": 4178.0, + "q50_diff": 5178.0, + "q75_diff": 106.94727000000057, + "q95_diff": 415.2753899999989 + }, + "P4_ST_LD": { + "q05_diff": 39.333740000000034, + "q25_diff": 76.64203000000003, + "q50_diff": 0.368720000000053, + "q75_diff": 2.007509999999968, + "q95_diff": 7.758240000000001 + }, + "P4_ST_PO": { + "q05_diff": 43.02301, + "q25_diff": 78.10687000000001, + "q50_diff": 0.6149300000000153, + "q75_diff": 2.1597300000000246, + "q95_diff": 7.8603200000000015 + }, + "P4_ST_PT01": { + "q05_diff": 108.0, + "q25_diff": 116.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P4_ST_TT01": { + "q05_diff": 22.0, + "q25_diff": 56.0, + "q50_diff": 1.2617199999986042, + "q75_diff": 1.0, + "q95_diff": 1.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0624831607922858, + "P1_B2016": 0.8239161706376917, + "P1_B3004": 0.9420514424699565, + "P1_B3005": 0.9755864990985265, + "P1_B4002": 0.9939428775878422, + "P1_B4005": 1.0901635813805555, + "P1_B400B": 1.000823178812837, + "P1_B4022": 1.0728287547128275, + "P1_FCV02Z": 1.0856923346443097, + "P1_FCV03D": 1.0287616800381814, + "P1_FCV03Z": 1.01920928123416, + "P1_FT01": 0.9097352918666402, + "P1_FT01Z": 0.912762944961449, + "P1_FT02": 0.9117761892474533, + "P1_FT02Z": 1.0572315062387077, + "P1_FT03": 0.9305900408187426, + "P1_FT03Z": 0.9861265787637634, + "P1_LCV01D": 0.9922897795117445, + "P1_LCV01Z": 0.9035160861736835, + "P1_LIT01": 0.8559220185741317, + "P1_PCV01D": 0.9847169107083952, + "P1_PCV01Z": 1.1483081333334428, + "P1_PCV02Z": 0.5764046458797007, + "P1_PIT01": 0.9302862520506019, + "P1_PIT02": 0.8133964033423414, + "P1_TIT01": 0.8348426617871705, + "P1_TIT02": 0.950787093191726, + "P2_24Vdc": 0.039965792976989475, + "P2_CO_rpm": 0.4019486663200069, + "P2_HILout": 0.24706059027478758, + "P2_MSD": 0.0, + "P2_SIT01": 0.49156029781639277, + "P2_SIT02": 0.763964286715772, + "P2_VT01": 0.3279489450854321, + "P2_VXT02": 0.6829469029404855, + "P2_VXT03": 0.7257564665587917, + "P2_VYT02": 0.708956109529326, + "P2_VYT03": 0.6451582481202596, + "P3_FIT01": 0.7874574045605387, + "P3_LCP01D": 1.0276466561226463, + "P3_LCV01D": 0.9954988909036026, + "P3_LIT01": 0.960072556685037, + "P3_PIT01": 0.8883587815956718, + "P4_HT_FD": 0.26400019639364447, + "P4_HT_LD": 1.052071148480924, + "P4_HT_PO": 1.0918064404320895, + "P4_LD": 0.9696178375756026, + "P4_ST_FD": 0.28595359004958487, + "P4_ST_GOV": 0.9454050192942649, + "P4_ST_LD": 0.9343184982665232, + "P4_ST_PO": 0.927849947228268, + "P4_ST_PT01": 0.8413419488661078, + "P4_ST_TT01": 0.8017488548689526 + }, + "discrete_jsd": { + "P1_FCV01D": 0.7708162008591284, + "P1_FCV01Z": 0.9973184330893802, + "P1_FCV02D": 0.7241545324106555, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.04436084816766643, + "P4_ST_PS": 0.12530984363905556 + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/full__seed1337/comprehensive_eval.json b/example/results/ablations/runs/full__seed1337/comprehensive_eval.json new file mode 100644 index 0000000..e4d5dd4 --- /dev/null +++ b/example/results/ablations/runs/full__seed1337/comprehensive_eval.json @@ -0,0 +1,537 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/full__seed1337/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.40165841194968555, + "avg_jsd": 0.028006235870934912, + "avg_lag1_diff": 0.29136546564037447 + }, + "two_sample": { + "continuous_mmd_rbf": 0.6529757976531982, + "continuous_mmd_gamma": 1.6922283487645608e-05, + "histogram_mmd_rbf": 0.6529804468154907, + "histogram_mmd_gamma": 1.692146470612109e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 162.5074005126953, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 2.945235307558582, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.21485304832458496, + "corr_frobenius": 16.57508087158203, + "lag1_corr_mean_abs_diff": 0.2144312560558319, + "lag1_corr_frobenius": 16.73354148864746, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.29505810141563416, + "corr_frobenius": 10.385537147521973, + "lag1_corr_mean_abs_diff": 0.3118370771408081, + "lag1_corr_frobenius": 10.768470764160156 + }, + "P2": { + "corr_mean_abs_diff": 0.12853267788887024, + "corr_frobenius": 2.545632839202881, + "lag1_corr_mean_abs_diff": 0.16028834879398346, + "lag1_corr_frobenius": 2.9215023517608643 + }, + "P3": { + "corr_mean_abs_diff": 0.4330995976924896, + "corr_frobenius": 2.9806668758392334, + "lag1_corr_mean_abs_diff": 0.488036572933197, + "lag1_corr_frobenius": 3.1031997203826904 + }, + "P4": { + "corr_mean_abs_diff": 0.18106727302074432, + "corr_frobenius": 2.683600425720215, + "lag1_corr_mean_abs_diff": 0.19239205121994019, + "lag1_corr_frobenius": 2.8247570991516113 + } + } + }, + "frequency": { + "avg_psd_l1": 0.019314780831336975, + "avg_psd_cosine": 0.33202425872099467, + "avg_low_high_ratio_abs_diff": 9433975.202625437 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 15.0, + "mean_dwell": 24.0, + "median_dwell": 2.5, + "mean_step": 68.42660395304362, + "median_step": 1.9043693542480469 + }, + "P1_B2004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B3004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.01115664489586733, + "median_step": 0.004810333251953125 + }, + "P1_B3005": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22998.14285714286, + "mean_abs_diff_mean_dwell": 14201.625685203559, + "mean_abs_diff_median_dwell": 14207.642857142857, + "mean_abs_diff_mean_step": 33.693117622162895, + "mean_abs_diff_median_step": 0.831573486328125 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 1.0, + "change_rate": 0.0, + "step_median": NaN + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.411041259765625, + "mean_abs_diff_change_rate": 0.4608201173921447 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.0026041666666666665, + "top1_mass": 1.0, + "top3_mass": 1.0, + "median_dwell": 384.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.90625, + "top1_mass": 0.028645833333333332, + "top3_mass": 0.06770833333333333, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.08333333333333333, + "top3_mass": 0.09114583333333333, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.0390625, + "top1_mass": 0.3463541666666667, + "top3_mass": 0.6744791666666666, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 1.0, + "top1_mass": 0.0026041666666666665, + "top3_mass": 0.0078125, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.3802083333333333, + "top1_mass": 0.13541666666666666, + "top3_mass": 0.2265625, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.9635416666666666, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.015625, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.1796875, + "top1_mass": 0.2552083333333333, + "top3_mass": 0.390625, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.521447499593099, + "mean_abs_diff_top1_mass": 0.11526870727539062, + "mean_abs_diff_top3_mass": 0.0891888936360677, + "mean_abs_diff_median_dwell": 47.75 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17563000321388245, + "q50": 0.17563000321388245, + "q95": 0.27557000517845154, + "tail_ratio": NaN + }, + "P2_SIT02": { + "q05": 774.910888671875, + "q50": 784.5538330078125, + "q95": 791.7017211914062, + "tail_ratio": 0.7412557756820052 + }, + "P1_FT03": { + "q05": 208.97262573242188, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.19263285679775694 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.590030978123347, + "mean_abs_diff_q50": 2.2669277489185333, + "mean_abs_diff_q95": 3.55855601032575, + "mean_abs_diff_tail_ratio": 0.04623891090189372 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 379.0, + "mean_dwell": 1.0105263157894737, + "median_dwell": 1.0, + "mean_step": 16.7416249780982, + "median_step": 7.720184326171875 + }, + "P1_FT02Z": { + "num_changes": 379.0, + "mean_dwell": 1.0105263157894737, + "median_dwell": 1.0, + "mean_step": 679.2698862558933, + "median_step": 195.1143798828125 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.0904041845427852, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 340.3546550414803, + "mean_abs_diff_median_step": 98.15037536621094 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 34.00083923339844, + "std": 31.541128158569336, + "lag1": 0.5380550026893616 + }, + "P2_24Vdc": { + "mean": 28.028627395629883, + "std": 0.0034831492230296135, + "lag1": 0.048770755529403687 + }, + "P2_HILout": { + "mean": 711.4832153320312, + "std": 9.425703048706055, + "lag1": 0.01359469536691904 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.2827949523925781, + "mean_abs_diff_std": 0.4692591578544428, + "mean_abs_diff_lag1": 0.2379635035370787 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9723039269447327, + "mae": 0.7831042408943176 + }, + "real_plus_synthetic": { + "rmse": 0.5518718361854553, + "mae": 0.3549392819404602 + }, + "rmse_ratio_synth_to_real": 1.7413850160137512 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8299860853432282, + "auprc": 0.5936877857343479, + "threshold": 0.012659437023103237, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6753246753246752, + "best_f1_threshold": 1.2799640893936157 + }, + "real_plus_synthetic": { + "auroc": 0.8843692022263452, + "auprc": 0.6436615501494315, + "threshold": 0.2907267212867737, + "f1": 0.23443223443223443, + "precision": 0.13973799126637554, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.5491346120834351 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/full__seed1337/comprehensive_eval_post.json b/example/results/ablations/runs/full__seed1337/comprehensive_eval_post.json new file mode 100644 index 0000000..1f78e85 --- /dev/null +++ b/example/results/ablations/runs/full__seed1337/comprehensive_eval_post.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/full__seed1337/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.21801376342773438, + "continuous_mmd_gamma": 5.3593154060743865e-05, + "histogram_mmd_rbf": 0.21808743476867676, + "histogram_mmd_gamma": 5.357758012673609e-05, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 89.01089477539062, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 1.6132067168804538, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.19314609467983246, + "corr_frobenius": 15.973841667175293, + "lag1_corr_mean_abs_diff": 0.20119889080524445, + "lag1_corr_frobenius": 16.674213409423828, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.3415932059288025, + "corr_frobenius": 11.884793281555176, + "lag1_corr_mean_abs_diff": 0.3632906377315521, + "lag1_corr_frobenius": 12.56336498260498 + }, + "P2": { + "corr_mean_abs_diff": 0.08792565762996674, + "corr_frobenius": 1.6836949586868286, + "lag1_corr_mean_abs_diff": 0.15461258590221405, + "lag1_corr_frobenius": 2.910283327102661 + }, + "P3": { + "corr_mean_abs_diff": 0.4330995976924896, + "corr_frobenius": 2.9806668758392334, + "lag1_corr_mean_abs_diff": 0.488036572933197, + "lag1_corr_frobenius": 3.1031997203826904 + }, + "P4": { + "corr_mean_abs_diff": 0.21049436926841736, + "corr_frobenius": 3.2200543880462646, + "lag1_corr_mean_abs_diff": 0.23382383584976196, + "lag1_corr_frobenius": 3.576234817504883 + } + } + }, + "frequency": { + "avg_psd_l1": 0.022376514971256256, + "avg_psd_cosine": 0.4402457082046653, + "avg_low_high_ratio_abs_diff": 18.461675871818574 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 308.0, + "mean_dwell": 1.2427184466019416, + "median_dwell": 1.0, + "mean_step": 0.8413416007896523, + "median_step": 0.7274894714355469 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 347.0, + "mean_dwell": 1.103448275862069, + "median_dwell": 1.0, + "mean_step": 41.3141472878254, + "median_step": 54.116028785705566 + }, + "P1_B2004": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 0.014068839402653235, + "median_step": 0.0017450004816055298 + }, + "P1_B3004": { + "num_changes": 370.0, + "mean_dwell": 1.0350404312668464, + "median_dwell": 1.0, + "mean_step": 14.77256890374261, + "median_step": 10.804779052734375 + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.8157603410765646, + "median_step": 0.5944099426269531 + }, + "P1_B3005": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 64.28072175926926, + "median_step": 41.228179931640625 + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22997.85714285714, + "mean_abs_diff_mean_dwell": 14417.281142645239, + "mean_abs_diff_median_dwell": 13988.57142857143, + "mean_abs_diff_mean_step": 19.03385039702909, + "mean_abs_diff_median_step": 16.840154950817425 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6822916666666666, + "change_rate": 0.7702349869451697, + "step_median": 68.81373023986816 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.09333292643229163, + "mean_abs_diff_change_rate": 0.309414869553025, + "mean_abs_diff_step_median": 68.78170993924141 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.015625, + "top1_mass": 0.40625, + "top3_mass": 0.7760416666666666, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8776041666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.059895833333333336, + "top3_mass": 0.0703125, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.18229166666666666, + "top3_mass": 0.3697916666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.8463541666666666, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4817708333333333, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.7864583333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3515625, + "top1_mass": 0.036458333333333336, + "top3_mass": 0.09635416666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5303484598795573, + "mean_abs_diff_top1_mass": 0.03112411499023438, + "mean_abs_diff_top3_mass": 0.05629857381184896, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17563000321388245, + "q50": 0.17563000321388245, + "q95": 0.27557000517845154, + "tail_ratio": NaN + }, + "P2_SIT02": { + "q05": 774.910888671875, + "q50": 784.5538330078125, + "q95": 791.7017211914062, + "tail_ratio": 0.7412557756820052 + }, + "P1_FT03": { + "q05": 208.97262573242188, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.19263285679775694 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.590030978123347, + "mean_abs_diff_q50": 2.2669277489185333, + "mean_abs_diff_q95": 3.55855601032575, + "mean_abs_diff_tail_ratio": 0.04623891090189372 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 64.74391126130756, + "median_step": 62.410614013671875 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 1312.3134692358592, + "median_step": 1284.6439208984375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.09039718638964156, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 680.877589673068, + "mean_abs_diff_median_step": 670.2603607177734 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 35.254512786865234, + "std": 31.650897979736328, + "lag1": -0.0927804708480835 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.003297113813459873, + "lag1": 0.06643600016832352 + }, + "P2_HILout": { + "mean": 711.4987182617188, + "std": 7.856828212738037, + "lag1": -0.013758387416601181 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.4565467834472656, + "mean_abs_diff_std": 0.2373291968833655, + "mean_abs_diff_lag1": 0.4579590285817782 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9636826515197754, + "mae": 0.7771188616752625 + }, + "real_plus_synthetic": { + "rmse": 0.5483431220054626, + "mae": 0.35133740305900574 + }, + "rmse_ratio_synth_to_real": 1.7259444120751009 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8286178107606678, + "auprc": 0.591504821106902, + "threshold": 0.001942009199410677, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6756756756756758, + "best_f1_threshold": 1.4600814580917358 + }, + "real_plus_synthetic": { + "auroc": 0.8840677179962895, + "auprc": 0.6460935090211736, + "threshold": 0.28852400183677673, + "f1": 0.24150943396226415, + "precision": 0.14479638009049775, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.5378937721252441 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/full__seed1337/eval.json b/example/results/ablations/runs/full__seed1337/eval.json new file mode 100644 index 0000000..a5bd20d --- /dev/null +++ b/example/results/ablations/runs/full__seed1337/eval.json @@ -0,0 +1,974 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.11384, + "std": 0.0 + }, + "P1_B2016": { + "mean": 1.3915169791666673, + "std": 0.13943831071397558 + }, + "P1_B3004": { + "mean": 396.00211, + "std": 0.0 + }, + "P1_B3005": { + "mean": 1015.43976, + "std": 0.0 + }, + "P1_B4002": { + "mean": 33.54375, + "std": 0.0 + }, + "P1_B4005": { + "mean": 0.0, + "std": 0.0 + }, + "P1_B400B": { + "mean": 1769.7853541666657, + "std": 1195.0342421744162 + }, + "P1_B4022": { + "mean": 36.881583541666664, + "std": 0.14105102171131376 + }, + "P1_FCV02Z": { + "mean": 22.320061927083323, + "std": 38.91077582497131 + }, + "P1_FCV03D": { + "mean": 58.570938776041665, + "std": 6.641066821382397 + }, + "P1_FCV03Z": { + "mean": 59.159972604166676, + "std": 6.782049383292698 + }, + "P1_FT01": { + "mean": 194.7738916406249, + "std": 32.02017013040693 + }, + "P1_FT01Z": { + "mean": 876.8946808333341, + "std": 84.82995107102002 + }, + "P1_FT02": { + "mean": 1129.006613541667, + "std": 890.9766838535422 + }, + "P1_FT02Z": { + "mean": 1129.006613541667, + "std": 890.9766838535422 + }, + "P1_FT03": { + "mean": 282.55279374999986, + "std": 33.70565422729771 + }, + "P1_FT03Z": { + "mean": 282.55279374999986, + "std": 33.70565422729771 + }, + "P1_LCV01D": { + "mean": 12.166996848958325, + "std": 3.0452708274616125 + }, + "P1_LCV01Z": { + "mean": 11.224420286458333, + "std": 3.3096185530935207 + }, + "P1_LIT01": { + "mean": 404.01072572916655, + "std": 15.830517116867311 + }, + "P1_PCV01D": { + "mean": 67.4503832552083, + "std": 23.852228142415264 + }, + "P1_PCV01Z": { + "mean": 68.30529460937498, + "std": 23.92581520523648 + }, + "P1_PCV02Z": { + "mean": 11.76605, + "std": 0.0 + }, + "P1_PIT01": { + "mean": 1.3641433593750003, + "std": 0.07412462833472319 + }, + "P1_PIT02": { + "mean": 0.22266562499999998, + "std": 0.050024260428937296 + }, + "P1_TIT01": { + "mean": 35.66191083333333, + "std": 0.587215446293234 + }, + "P1_TIT02": { + "mean": 36.49771968750001, + "std": 1.2278381103056584 + }, + "P2_24Vdc": { + "mean": 28.02862885416666, + "std": 0.0034830646209355787 + }, + "P2_CO_rpm": { + "mean": 54102.28948968744, + "std": 24.102981208861518 + }, + "P2_HILout": { + "mean": 711.4832500260413, + "std": 9.425702432265151 + }, + "P2_MSD": { + "mean": 758.4425, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.1080989583335, + "std": 5.823213497430518 + }, + "P2_SIT02": { + "mean": 783.9254949999996, + "std": 5.841862921294127 + }, + "P2_VT01": { + "mean": 11.923772838541666, + "std": 0.06900960879865246 + }, + "P2_VXT02": { + "mean": -3.2993482031250005, + "std": 0.4773366739873613 + }, + "P2_VXT03": { + "mean": -1.305684166666667, + "std": 0.3573219073509669 + }, + "P2_VYT02": { + "mean": 3.980818906249999, + "std": 0.5058086972940764 + }, + "P2_VYT03": { + "mean": 6.233472864583333, + "std": 0.38911584026721935 + }, + "P3_FIT01": { + "mean": 899.7896354166668, + "std": 1625.0289852427827 + }, + "P3_LCP01D": { + "mean": 4590.882869270832, + "std": 5143.432130733786 + }, + "P3_LCV01D": { + "mean": 7122.205779010418, + "std": 6859.721607840774 + }, + "P3_LIT01": { + "mean": 13352.467730182292, + "std": 4371.257796553593 + }, + "P3_PIT01": { + "mean": 45.75260416666667, + "std": 484.2709130492731 + }, + "P4_HT_FD": { + "mean": -0.0005401822916666672, + "std": 0.0045261268484950035 + }, + "P4_HT_LD": { + "mean": -115.69799739583333, + "std": 35.27327640165136 + }, + "P4_HT_PO": { + "mean": 34.00084049479167, + "std": 31.541126911615113 + }, + "P4_LD": { + "mean": 366.1261522135412, + "std": 61.030691338163635 + }, + "P4_ST_FD": { + "mean": -6.567708333333341e-05, + "std": 0.0040457343761286235 + }, + "P4_ST_GOV": { + "mean": 17918.88347888019, + "std": 1897.796541489009 + }, + "P4_ST_LD": { + "mean": 332.09558960937517, + "std": 38.601298687602366 + }, + "P4_ST_PO": { + "mean": 332.36973, + "std": 35.263965506582586 + }, + "P4_ST_PT01": { + "mean": 10047.419270833343, + "std": 14.88962852414534 + }, + "P4_ST_TT01": { + "mean": 27608.260976171874, + "std": 24.748102592251026 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.020272347909024324, + "std_abs_err": 0.017998437082092405 + }, + "P1_B2016": { + "mean_abs_err": 0.0002506545019003159, + "std_abs_err": 0.0036647482374125018 + }, + "P1_B3004": { + "mean_abs_err": 7.383953824526145, + "std_abs_err": 15.41814958804181 + }, + "P1_B3005": { + "mean_abs_err": 45.609026528629784, + "std_abs_err": 61.18331651100028 + }, + "P1_B4002": { + "mean_abs_err": 1.2268236801201198, + "std_abs_err": 0.5786734991990778 + }, + "P1_B4005": { + "mean_abs_err": 60.52510701619792, + "std_abs_err": 42.18129744770969 + }, + "P1_B400B": { + "mean_abs_err": 8.690516011302407, + "std_abs_err": 1.6819692553435743 + }, + "P1_B4022": { + "mean_abs_err": 0.8204088942641405, + "std_abs_err": 0.5303998331235046 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.6308729733889535, + "std_abs_err": 1.52770961428687 + }, + "P1_FCV03D": { + "mean_abs_err": 0.003075325194835443, + "std_abs_err": 0.04157110267364228 + }, + "P1_FCV03Z": { + "mean_abs_err": 0.1390665760231613, + "std_abs_err": 0.06724541896109582 + }, + "P1_FT01": { + "mean_abs_err": 0.05023728434463237, + "std_abs_err": 4.672188291762605 + }, + "P1_FT01Z": { + "mean_abs_err": 2.155000670818481, + "std_abs_err": 9.064248107701772 + }, + "P1_FT02": { + "mean_abs_err": 3.811539692959059, + "std_abs_err": 1.780520002129606 + }, + "P1_FT02Z": { + "mean_abs_err": 649.4718330669816, + "std_abs_err": 302.36867191760734 + }, + "P1_FT03": { + "mean_abs_err": 0.08617652622905325, + "std_abs_err": 0.23144111600220185 + }, + "P1_FT03Z": { + "mean_abs_err": 778.4892445985217, + "std_abs_err": 27.41983463631962 + }, + "P1_LCV01D": { + "mean_abs_err": 0.012664187183188957, + "std_abs_err": 0.13065587443149118 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.7416537428795849, + "std_abs_err": 0.3461490557482092 + }, + "P1_LIT01": { + "mean_abs_err": 0.10414643383973043, + "std_abs_err": 0.19861906849333977 + }, + "P1_PCV01D": { + "mean_abs_err": 0.015190480483084912, + "std_abs_err": 0.08246144150968959 + }, + "P1_PCV01Z": { + "mean_abs_err": 0.10285718752760431, + "std_abs_err": 0.06121539457283376 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.07670294660492694, + "std_abs_err": 0.09684446546464737 + }, + "P1_PIT01": { + "mean_abs_err": 0.013337861378391658, + "std_abs_err": 0.01269206896484109 + }, + "P1_PIT02": { + "mean_abs_err": 0.26449146534151674, + "std_abs_err": 0.4486873776259425 + }, + "P1_TIT01": { + "mean_abs_err": 0.019297409099884533, + "std_abs_err": 0.0039348191072849925 + }, + "P1_TIT02": { + "mean_abs_err": 0.027258990836521946, + "std_abs_err": 0.006079384281395317 + }, + "P2_24Vdc": { + "mean_abs_err": 8.084846101041876e-05, + "std_abs_err": 0.00014059850652866366 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.42070819523360115, + "std_abs_err": 1.647249434073153 + }, + "P2_HILout": { + "mean_abs_err": 0.020440755945173805, + "std_abs_err": 0.9709765550089884 + }, + "P2_MSD": { + "mean_abs_err": 4.750739999999951, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.7958184551094973, + "std_abs_err": 0.624298136986293 + }, + "P2_SIT02": { + "mean_abs_err": 0.02175711745337594, + "std_abs_err": 0.6512831644811605 + }, + "P2_VT01": { + "mean_abs_err": 4.280746846774264e-05, + "std_abs_err": 0.0004959267266504003 + }, + "P2_VXT02": { + "mean_abs_err": 0.0008302692649957422, + "std_abs_err": 0.006303747716141006 + }, + "P2_VXT03": { + "mean_abs_err": 0.0012091199980108769, + "std_abs_err": 0.007694528187313254 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006499437429381416, + "std_abs_err": 0.005898244301288547 + }, + "P2_VYT03": { + "mean_abs_err": 0.0009322482145517341, + "std_abs_err": 0.009312878296127758 + }, + "P3_FIT01": { + "mean_abs_err": 251.11054068951637, + "std_abs_err": 154.69571254749212 + }, + "P3_LCP01D": { + "mean_abs_err": 6.155721030258064, + "std_abs_err": 13.43213301790729 + }, + "P3_LCV01D": { + "mean_abs_err": 11.670937970823616, + "std_abs_err": 19.793712024952583 + }, + "P3_LIT01": { + "mean_abs_err": 2.3767625116943236, + "std_abs_err": 17.083159625656663 + }, + "P3_PIT01": { + "mean_abs_err": 603.6052988566611, + "std_abs_err": 666.6510176579222 + }, + "P4_HT_FD": { + "mean_abs_err": 0.00043699434631492743, + "std_abs_err": 0.002494108534090903 + }, + "P4_HT_LD": { + "mean_abs_err": 149.69291447551893, + "std_abs_err": 1.8494828916530537 + }, + "P4_HT_PO": { + "mean_abs_err": 0.017032700034761206, + "std_abs_err": 0.10025569429363657 + }, + "P4_LD": { + "mean_abs_err": 0.002601580653106339, + "std_abs_err": 0.41488496872062797 + }, + "P4_ST_FD": { + "mean_abs_err": 3.65638534501846e-05, + "std_abs_err": 0.0024035581181385945 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.922217632665706, + "std_abs_err": 45.65332550123571 + }, + "P4_ST_LD": { + "mean_abs_err": 0.09269243955748152, + "std_abs_err": 0.7481293959848188 + }, + "P4_ST_PO": { + "mean_abs_err": 0.10075867634122915, + "std_abs_err": 0.8130220041742859 + }, + "P4_ST_PT01": { + "mean_abs_err": 3.1558859857395873, + "std_abs_err": 2.8951415011737893 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.7717155076570634, + "std_abs_err": 0.8307807876826985 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 1.0, + "P1_B2016": 0.04343750000000002, + "P1_B3004": 0.8108, + "P1_B3005": 0.8106, + "P1_B4002": 0.81055, + "P1_B4005": 0.9113, + "P1_B400B": 0.4596875, + "P1_B4022": 0.5730333333333334, + "P1_FCV02Z": 0.6912666666666666, + "P1_FCV03D": 0.48778750000000004, + "P1_FCV03Z": 0.4924416666666667, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4726416666666667, + "P1_FT02": 0.4587083333333334, + "P1_FT02Z": 0.8361000000000001, + "P1_FT03": 0.5241, + "P1_FT03Z": 1.0, + "P1_LCV01D": 0.4479875, + "P1_LCV01Z": 0.4260125, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5573083333333333, + "P1_PCV01Z": 0.5560208333333334, + "P1_PCV02Z": 1.0, + "P1_PIT01": 0.15733333333333333, + "P1_PIT02": 0.61435, + "P1_TIT01": 0.09973333333333334, + "P1_TIT02": 0.10090833333333338, + "P2_24Vdc": 0.12050416666666663, + "P2_CO_rpm": 0.09332916666666669, + "P2_HILout": 0.08234999999999998, + "P2_MSD": 1.0, + "P2_SIT01": 0.4626833333333333, + "P2_SIT02": 0.5013166666666666, + "P2_VT01": 0.12326250000000005, + "P2_VXT02": 0.28197916666666667, + "P2_VXT03": 0.43657916666666663, + "P2_VYT02": 0.2505666666666666, + "P2_VYT03": 0.2697583333333333, + "P3_FIT01": 0.18958333333333338, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.08329166666666671, + "P3_LIT01": 0.09349166666666664, + "P3_PIT01": 0.4136708333333333, + "P4_HT_FD": 0.11226666666666663, + "P4_HT_LD": 0.9375, + "P4_HT_PO": 0.08826666666666669, + "P4_LD": 0.031079166666666658, + "P4_ST_FD": 0.03345833333333331, + "P4_ST_GOV": 0.03512083333333338, + "P4_ST_LD": 0.032354166666666684, + "P4_ST_PO": 0.03597499999999998, + "P4_ST_PT01": 0.16995416666666663, + "P4_ST_TT01": 0.08401666666666663 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.08333, + "q25_diff": 0.012630000000000002, + "q50_diff": 0.012630000000000002, + "q75_diff": 0.012630000000000002, + "q95_diff": 0.012630000000000002 + }, + "P1_B2016": { + "q05_diff": 0.011969999999999814, + "q25_diff": 0.010260000000000158, + "q50_diff": 0.005159999999999831, + "q75_diff": 0.014349999999999863, + "q95_diff": 0.029469999999999885 + }, + "P1_B3004": { + "q05_diff": 2.8398500000000126, + "q25_diff": 1.6357399999999984, + "q50_diff": 1.6357399999999984, + "q75_diff": 1.6357399999999984, + "q95_diff": 1.6357399999999984 + }, + "P1_B3005": { + "q05_diff": 13.44177000000002, + "q25_diff": 13.44177000000002, + "q50_diff": 13.44177000000002, + "q75_diff": 13.44177000000002, + "q95_diff": 99.67999000000009 + }, + "P1_B4002": { + "q05_diff": 1.5437500000000028, + "q25_diff": 0.11175000000000068, + "q50_diff": 0.11175000000000068, + "q75_diff": 0.11175000000000068, + "q95_diff": 0.11175000000000068 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 100.0, + "q50_diff": 100.0, + "q75_diff": 100.0, + "q95_diff": 100.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2480.00824, + "q50_diff": 252.20629999999983, + "q75_diff": 9.545890000000327, + "q95_diff": 7.159419999999955 + }, + "P1_B4022": { + "q05_diff": 1.2294699999999992, + "q25_diff": 0.1574500000000043, + "q50_diff": 0.4875299999999996, + "q75_diff": 0.5946499999999943, + "q95_diff": 0.6461800000000011 + }, + "P1_FCV02Z": { + "q05_diff": 0.015249999999999986, + "q25_diff": 0.007630000000000026, + "q50_diff": 0.06103999999999998, + "q75_diff": 44.33441, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 2.891619999999996, + "q25_diff": 0.10130999999999801, + "q50_diff": 8.5484, + "q75_diff": 11.14537, + "q95_diff": 0.5946799999999968 + }, + "P1_FCV03Z": { + "q05_diff": 3.2806399999999982, + "q25_diff": 0.5874600000000001, + "q50_diff": 8.96454, + "q75_diff": 11.566150000000007, + "q95_diff": 0.5950900000000132 + }, + "P1_FT01": { + "q05_diff": 7.9056500000000085, + "q25_diff": 8.615820000000014, + "q50_diff": 32.61562999999998, + "q75_diff": 37.574780000000004, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 44.013919999999985, + "q25_diff": 17.829899999999952, + "q50_diff": 74.79840000000002, + "q75_diff": 84.80078000000003, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.2434799999999, + "q50_diff": 258.05504999999994, + "q75_diff": 13.160759999999982, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 27.82217, + "q25_diff": 2761.18427, + "q50_diff": 1117.6293999999998, + "q75_diff": 871.0228200000001, + "q95_diff": 865.2337600000001 + }, + "P1_FT03": { + "q05_diff": 36.13194999999999, + "q25_diff": 1.5291600000000187, + "q50_diff": 51.52633000000003, + "q75_diff": 59.53728000000001, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 788.86581, + "q25_diff": 752.66955, + "q50_diff": 703.89301, + "q75_diff": 697.8350800000001, + "q95_diff": 800.17028 + }, + "P1_LCV01D": { + "q05_diff": 0.10856999999999939, + "q25_diff": 1.20613, + "q50_diff": 4.63339, + "q75_diff": 3.31133, + "q95_diff": 0.619250000000001 + }, + "P1_LCV01Z": { + "q05_diff": 0.5874600000000001, + "q25_diff": 1.1901799999999998, + "q50_diff": 1.3122600000000002, + "q75_diff": 3.387459999999999, + "q95_diff": 0.7096700000000009 + }, + "P1_LIT01": { + "q05_diff": 14.301179999999988, + "q25_diff": 0.2044700000000148, + "q50_diff": 6.23626999999999, + "q75_diff": 8.229829999999993, + "q95_diff": 34.29950000000002 + }, + "P1_PCV01D": { + "q05_diff": 1.0659799999999962, + "q25_diff": 3.7112199999999973, + "q50_diff": 22.24991, + "q75_diff": 44.864110000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.9636600000000044, + "q25_diff": 3.814699999999995, + "q50_diff": 22.722420000000007, + "q75_diff": 45.46356000000001, + "q95_diff": 45.144450000000006 + }, + "P1_PCV02Z": { + "q05_diff": 0.2441399999999998, + "q25_diff": 0.2441399999999998, + "q50_diff": 0.2517700000000005, + "q75_diff": 0.2517700000000005, + "q95_diff": 0.2517700000000005 + }, + "P1_PIT01": { + "q05_diff": 0.029750000000000165, + "q25_diff": 0.011450000000000182, + "q50_diff": 0.009149999999999991, + "q75_diff": 0.022119999999999918, + "q95_diff": 0.035090000000000066 + }, + "P1_PIT02": { + "q05_diff": 0.028989999999999988, + "q25_diff": 0.09993999999999997, + "q50_diff": 0.10070999999999997, + "q75_diff": 0.0015300000000000313, + "q95_diff": 0.0022900000000000142 + }, + "P1_TIT01": { + "q05_diff": 0.045769999999997424, + "q25_diff": 0.07629999999999626, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.24206000000000216, + "q95_diff": 0.09154999999999802 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.030520000000002767, + "q50_diff": 0.09154999999999802, + "q75_diff": 0.30516999999999683, + "q95_diff": 0.4297200000000032 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 7.000000000090267e-05, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.000920000000000698 + }, + "P2_CO_rpm": { + "q05_diff": 11.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 2.0, + "q95_diff": 1.0 + }, + "P2_HILout": { + "q05_diff": 0.5523099999999204, + "q25_diff": 2.020259999999894, + "q50_diff": 0.08911000000000513, + "q75_diff": 1.5326000000000022, + "q95_diff": 1.1968400000000656 + }, + "P2_MSD": { + "q05_diff": 4.750739999999951, + "q25_diff": 4.750739999999951, + "q50_diff": 4.750739999999951, + "q75_diff": 4.750739999999951, + "q95_diff": 4.750739999999951 + }, + "P2_SIT01": { + "q05_diff": 1.0, + "q25_diff": 3.0, + "q50_diff": 5.0, + "q75_diff": 6.0, + "q95_diff": 6.0 + }, + "P2_SIT02": { + "q05_diff": 2.650700000000029, + "q25_diff": 3.569340000000011, + "q50_diff": 5.613770000000045, + "q75_diff": 6.357240000000047, + "q95_diff": 6.842530000000011 + }, + "P2_VT01": { + "q05_diff": 0.008340000000000458, + "q25_diff": 0.015269999999999229, + "q50_diff": 0.013980000000000103, + "q75_diff": 0.01343999999999923, + "q95_diff": 0.018990000000000506 + }, + "P2_VXT02": { + "q05_diff": 0.13161000000000023, + "q25_diff": 0.29940999999999995, + "q50_diff": 0.28532, + "q75_diff": 0.2669400000000004, + "q95_diff": 0.45862999999999987 + }, + "P2_VXT03": { + "q05_diff": 0.09976000000000007, + "q25_diff": 0.26910999999999996, + "q50_diff": 0.2972699999999999, + "q75_diff": 0.3267199999999999, + "q95_diff": 0.60987 + }, + "P2_VYT02": { + "q05_diff": 0.09552000000000005, + "q25_diff": 0.2620300000000002, + "q50_diff": 0.22682000000000002, + "q75_diff": 0.23567999999999945, + "q95_diff": 0.3888600000000002 + }, + "P2_VYT03": { + "q05_diff": 0.056390000000000384, + "q25_diff": 0.1858500000000003, + "q50_diff": 0.17562999999999995, + "q75_diff": 0.18247999999999998, + "q95_diff": 0.35470999999999986 + }, + "P3_FIT01": { + "q05_diff": 1.0, + "q25_diff": 0.0, + "q50_diff": 72.0, + "q75_diff": 2010.0, + "q95_diff": 63.0 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 88.22606999999994, + "q75_diff": 131.70507999999973, + "q95_diff": 10.988279999999577 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 256.0, + "q50_diff": 3227.02344, + "q75_diff": 192.0, + "q95_diff": 31.28319999999985 + }, + "P3_LIT01": { + "q05_diff": 1131.2470700000003, + "q25_diff": 28.58984000000055, + "q50_diff": 649.9677699999993, + "q75_diff": 1146.6367199999986, + "q95_diff": 1.662110000001121 + }, + "P3_PIT01": { + "q05_diff": 4.0, + "q25_diff": 5.0, + "q50_diff": 0.0, + "q75_diff": 928.0, + "q95_diff": 3524.0 + }, + "P4_HT_FD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 7e-05, + "q95_diff": 0.0 + }, + "P4_HT_LD": { + "q05_diff": 124.7862, + "q25_diff": 124.7862, + "q50_diff": 179.53157, + "q75_diff": 193.36982, + "q95_diff": 56.57957999999999 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 1.24783, + "q50_diff": 14.723600000000005, + "q75_diff": 0.8129899999999992, + "q95_diff": 0.08902000000000498 + }, + "P4_LD": { + "q05_diff": 0.5058300000000031, + "q25_diff": 3.0157100000000128, + "q50_diff": 3.449520000000007, + "q75_diff": 0.16302000000001726, + "q95_diff": 4.291780000000017 + }, + "P4_ST_FD": { + "q05_diff": 4.99999999999997e-05, + "q25_diff": 0.00013999999999999993, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.00014999999999999996 + }, + "P4_ST_GOV": { + "q05_diff": 125.97559000000001, + "q25_diff": 38.71679999999833, + "q50_diff": 30.52147999999943, + "q75_diff": 105.58594000000085, + "q95_diff": 402.84179999999833 + }, + "P4_ST_LD": { + "q05_diff": 0.5073900000000435, + "q25_diff": 0.40126999999995405, + "q50_diff": 0.4448600000000056, + "q75_diff": 2.0440699999999765, + "q95_diff": 7.88191999999998 + }, + "P4_ST_PO": { + "q05_diff": 2.2981300000000147, + "q25_diff": 0.7247299999999655, + "q50_diff": 0.6087400000000116, + "q75_diff": 2.2022400000000175, + "q95_diff": 7.818750000000023 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 20.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 30.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 7.0, + "q50_diff": 1.7675799999997253, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0189476811306442e-05, + "P1_B2016": 0.09372153962205165, + "P1_B3004": 1.018740504876714e-05, + "P1_B3005": 0.9999898125942444, + "P1_B4002": 0.9999898146591784, + "P1_B4005": 0.9999997508214005, + "P1_B400B": 0.4439571613343968, + "P1_B4022": 0.03037469907643997, + "P1_FCV02Z": 0.05990495226799131, + "P1_FCV03D": 0.27813943305114686, + "P1_FCV03Z": 0.1319412848524787, + "P1_FT01": 0.6609997822834346, + "P1_FT01Z": 0.3575180502180102, + "P1_FT02": 0.6690460556684432, + "P1_FT02Z": 0.6691010342783548, + "P1_FT03": 0.3022907751961552, + "P1_FT03Z": 0.30215371815154835, + "P1_LCV01D": 0.2945091789081814, + "P1_LCV01Z": 0.2660150572155058, + "P1_LIT01": 0.14869229177703291, + "P1_PCV01D": 0.23625611920865597, + "P1_PCV01Z": 0.10360200752869408, + "P1_PCV02Z": 0.5669400491575541, + "P1_PIT01": 0.2884406396378245, + "P1_PIT02": 0.16563474008732848, + "P1_TIT01": 0.043653145904043456, + "P1_TIT02": 0.08140759059784364, + "P2_24Vdc": 0.0222998336628301, + "P2_CO_rpm": 0.41590072608765727, + "P2_HILout": 0.21970749804238027, + "P2_MSD": 0.0, + "P2_SIT01": 0.49280073603354146, + "P2_SIT02": 0.5608912244785349, + "P2_VT01": 0.14955850806894322, + "P2_VXT02": 0.1622295159815592, + "P2_VXT03": 0.19162944594539777, + "P2_VYT02": 0.15176778852528572, + "P2_VYT03": 0.1718451687373067, + "P3_FIT01": 0.16161011867720687, + "P3_LCP01D": 0.14050969105480315, + "P3_LCV01D": 0.347878557710531, + "P3_LIT01": 0.07579325279870874, + "P3_PIT01": 0.7237460833164835, + "P4_HT_FD": 0.05083342513502456, + "P4_HT_LD": 0.15670838558697175, + "P4_HT_PO": 0.46097106172696245, + "P4_LD": 0.06981743984686695, + "P4_ST_FD": 0.2809423474777602, + "P4_ST_GOV": 0.11942940208112951, + "P4_ST_LD": 0.15564231905790638, + "P4_ST_PO": 0.1019794048923851, + "P4_ST_PT01": 0.8083994035521874, + "P4_ST_TT01": 0.05517927948168333 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.5505791762707029, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + }, + "avg_ks": 0.40165841194968555, + "avg_jsd": 0.028006235870934912, + "avg_lag1_diff": 0.29136546564037447 +} \ No newline at end of file diff --git a/example/results/ablations/runs/full__seed1337/eval_post.json b/example/results/ablations/runs/full__seed1337/eval_post.json new file mode 100644 index 0000000..491746f --- /dev/null +++ b/example/results/ablations/runs/full__seed1337/eval_post.json @@ -0,0 +1,971 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09355486979166669, + "std": 0.01799194293084728 + }, + "P1_B2016": { + "mean": 1.3915169791666673, + "std": 0.13943831071397558 + }, + "P1_B3004": { + "mean": 403.2905014583335, + "std": 14.217563527951576 + }, + "P1_B3005": { + "mean": 1059.5253898958326, + "std": 58.42587703751758 + }, + "P1_B4002": { + "mean": 32.37490617187499, + "std": 0.6519533213854061 + }, + "P1_B4005": { + "mean": 64.99571291666655, + "std": 42.03467331899641 + }, + "P1_B400B": { + "mean": 1769.7853541666657, + "std": 1195.0342421744162 + }, + "P1_B4022": { + "mean": 36.08671476562498, + "std": 0.7147782383668128 + }, + "P1_FCV02Z": { + "mean": 21.954832630208315, + "std": 38.87335152465215 + }, + "P1_FCV03D": { + "mean": 58.14541601562505, + "std": 6.267267768996455 + }, + "P1_FCV03Z": { + "mean": 58.21398343750003, + "std": 6.423211358701469 + }, + "P1_FT01": { + "mean": 194.7738916406249, + "std": 32.02017013040693 + }, + "P1_FT01Z": { + "mean": 876.8946808333341, + "std": 84.82995107102002 + }, + "P1_FT02": { + "mean": 1129.006613541667, + "std": 890.9766838535422 + }, + "P1_FT02Z": { + "mean": 1843.7261292187509, + "std": 1190.4241853713818 + }, + "P1_FT03": { + "mean": 282.55279374999986, + "std": 33.70565422729771 + }, + "P1_FT03Z": { + "mean": 1058.2564329947922, + "std": 59.50957255113764 + }, + "P1_LCV01D": { + "mean": 11.699055963541667, + "std": 3.0155193220433567 + }, + "P1_LCV01Z": { + "mean": 11.627510442708333, + "std": 3.0251072332459787 + }, + "P1_LIT01": { + "mean": 404.01072572916655, + "std": 15.830517116867311 + }, + "P1_PCV01D": { + "mean": 65.64593611979173, + "std": 23.56629129628467 + }, + "P1_PCV01Z": { + "mean": 65.565628671875, + "std": 23.790956943468952 + }, + "P1_PCV02Z": { + "mean": 11.867279765625007, + "std": 0.10811550492611105 + }, + "P1_PIT01": { + "mean": 1.3641433593750003, + "std": 0.07412462833472319 + }, + "P1_PIT02": { + "mean": 0.22266562499999998, + "std": 0.050024260428937296 + }, + "P1_TIT01": { + "mean": 35.66191083333333, + "std": 0.587215446293234 + }, + "P1_TIT02": { + "mean": 36.49771968750001, + "std": 1.2278381103056584 + }, + "P2_24Vdc": { + "mean": 28.02860809895833, + "std": 0.0032970984602375326 + }, + "P2_CO_rpm": { + "mean": 54102.28948968744, + "std": 24.102981208861518 + }, + "P2_HILout": { + "mean": 711.4986896354167, + "std": 7.856828438379631 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.1080989583335, + "std": 5.823213497430518 + }, + "P2_SIT02": { + "mean": 783.9254949999996, + "std": 5.841862921294127 + }, + "P2_VT01": { + "mean": 11.923772838541666, + "std": 0.06900960879865246 + }, + "P2_VXT02": { + "mean": -3.2993482031250005, + "std": 0.4773366739873613 + }, + "P2_VXT03": { + "mean": -1.305684166666667, + "std": 0.3573219073509669 + }, + "P2_VYT02": { + "mean": 3.980818906249999, + "std": 0.5058086972940764 + }, + "P2_VYT03": { + "mean": 6.233472864583333, + "std": 0.38911584026721935 + }, + "P3_FIT01": { + "mean": 899.7896354166668, + "std": 1625.0289852427827 + }, + "P3_LCP01D": { + "mean": 4590.882869270832, + "std": 5143.432130733786 + }, + "P3_LCV01D": { + "mean": 7122.205779010418, + "std": 6859.721607840774 + }, + "P3_LIT01": { + "mean": 13352.467730182292, + "std": 4371.257796553593 + }, + "P3_PIT01": { + "mean": 45.75260416666667, + "std": 484.2709130492731 + }, + "P4_HT_FD": { + "mean": -0.0005401822916666672, + "std": 0.0045261268484950035 + }, + "P4_HT_LD": { + "mean": 34.231568880208286, + "std": 33.60361854009684 + }, + "P4_HT_PO": { + "mean": 35.25451283854166, + "std": 31.650898099038645 + }, + "P4_LD": { + "mean": 366.1261522135412, + "std": 61.030691338163635 + }, + "P4_ST_FD": { + "mean": -6.567708333333341e-05, + "std": 0.0040457343761286235 + }, + "P4_ST_GOV": { + "mean": 17918.88347888019, + "std": 1897.796541489009 + }, + "P4_ST_LD": { + "mean": 332.09558960937517, + "std": 38.601298687602366 + }, + "P4_ST_PO": { + "mean": 332.36973, + "std": 35.263965506582586 + }, + "P4_ST_PT01": { + "mean": 10047.419270833343, + "std": 14.88962852414534 + }, + "P4_ST_TT01": { + "mean": 27608.260976171874, + "std": 24.748102592251026 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 1.2782299308983225e-05, + "std_abs_err": 6.494151245123314e-06 + }, + "P1_B2016": { + "mean_abs_err": 0.0002506545019003159, + "std_abs_err": 0.0036647482374125018 + }, + "P1_B3004": { + "mean_abs_err": 0.09556236619266656, + "std_abs_err": 1.2005860600902345 + }, + "P1_B3005": { + "mean_abs_err": 1.5233966327971302, + "std_abs_err": 2.757439473482698 + }, + "P1_B4002": { + "mean_abs_err": 0.05797985199510691, + "std_abs_err": 0.07327982218632823 + }, + "P1_B4005": { + "mean_abs_err": 4.470605900468627, + "std_abs_err": 0.14662412871327746 + }, + "P1_B400B": { + "mean_abs_err": 8.690516011302407, + "std_abs_err": 1.6819692553435743 + }, + "P1_B4022": { + "mean_abs_err": 0.02554011822245883, + "std_abs_err": 0.04332738353199439 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.996102270263961, + "std_abs_err": 1.5651339146060295 + }, + "P1_FCV03D": { + "mean_abs_err": 0.42244743522177686, + "std_abs_err": 0.3322279497122995 + }, + "P1_FCV03Z": { + "mean_abs_err": 1.0850557426898035, + "std_abs_err": 0.29159260563013323 + }, + "P1_FT01": { + "mean_abs_err": 0.05023728434463237, + "std_abs_err": 4.672188291762605 + }, + "P1_FT01Z": { + "mean_abs_err": 2.155000670818481, + "std_abs_err": 9.064248107701772 + }, + "P1_FT02": { + "mean_abs_err": 3.811539692959059, + "std_abs_err": 1.780520002129606 + }, + "P1_FT02Z": { + "mean_abs_err": 65.24768261010217, + "std_abs_err": 2.9211703997677887 + }, + "P1_FT03": { + "mean_abs_err": 0.08617652622905325, + "std_abs_err": 0.23144111600220185 + }, + "P1_FT03Z": { + "mean_abs_err": 2.785605353729352, + "std_abs_err": 1.6159163124796905 + }, + "P1_LCV01D": { + "mean_abs_err": 0.45527669823346884, + "std_abs_err": 0.1009043690132354 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.33856358662958463, + "std_abs_err": 0.061637735900667234 + }, + "P1_LIT01": { + "mean_abs_err": 0.10414643383973043, + "std_abs_err": 0.19861906849333977 + }, + "P1_PCV01D": { + "mean_abs_err": 1.8196376158996515, + "std_abs_err": 0.20347540462090308 + }, + "P1_PCV01Z": { + "mean_abs_err": 2.842523125027583, + "std_abs_err": 0.07364286719469249 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.024526819020080026, + "std_abs_err": 0.011271039461463683 + }, + "P1_PIT01": { + "mean_abs_err": 0.013337861378391658, + "std_abs_err": 0.01269206896484109 + }, + "P1_PIT02": { + "mean_abs_err": 0.26449146534151674, + "std_abs_err": 0.4486873776259425 + }, + "P1_TIT01": { + "mean_abs_err": 0.019297409099884533, + "std_abs_err": 0.0039348191072849925 + }, + "P1_TIT02": { + "mean_abs_err": 0.027258990836521946, + "std_abs_err": 0.006079384281395317 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00010160366933931186, + "std_abs_err": 4.536765416938244e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.42070819523360115, + "std_abs_err": 1.647249434073153 + }, + "P2_HILout": { + "mean_abs_err": 0.035880365320508645, + "std_abs_err": 0.5978974388765312 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.7958184551094973, + "std_abs_err": 0.624298136986293 + }, + "P2_SIT02": { + "mean_abs_err": 0.02175711745337594, + "std_abs_err": 0.6512831644811605 + }, + "P2_VT01": { + "mean_abs_err": 4.280746846774264e-05, + "std_abs_err": 0.0004959267266504003 + }, + "P2_VXT02": { + "mean_abs_err": 0.0008302692649957422, + "std_abs_err": 0.006303747716141006 + }, + "P2_VXT03": { + "mean_abs_err": 0.0012091199980108769, + "std_abs_err": 0.007694528187313254 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006499437429381416, + "std_abs_err": 0.005898244301288547 + }, + "P2_VYT03": { + "mean_abs_err": 0.0009322482145517341, + "std_abs_err": 0.009312878296127758 + }, + "P3_FIT01": { + "mean_abs_err": 251.11054068951637, + "std_abs_err": 154.69571254749212 + }, + "P3_LCP01D": { + "mean_abs_err": 6.155721030258064, + "std_abs_err": 13.43213301790729 + }, + "P3_LCV01D": { + "mean_abs_err": 11.670937970823616, + "std_abs_err": 19.793712024952583 + }, + "P3_LIT01": { + "mean_abs_err": 2.3767625116943236, + "std_abs_err": 17.083159625656663 + }, + "P3_PIT01": { + "mean_abs_err": 603.6052988566611, + "std_abs_err": 666.6510176579222 + }, + "P4_HT_FD": { + "mean_abs_err": 0.00043699434631492743, + "std_abs_err": 0.002494108534090903 + }, + "P4_HT_LD": { + "mean_abs_err": 0.23665180052267232, + "std_abs_err": 0.17982503009853446 + }, + "P4_HT_PO": { + "mean_abs_err": 1.2707050437847514, + "std_abs_err": 0.21002688171716954 + }, + "P4_LD": { + "mean_abs_err": 0.002601580653106339, + "std_abs_err": 0.41488496872062797 + }, + "P4_ST_FD": { + "mean_abs_err": 3.65638534501846e-05, + "std_abs_err": 0.0024035581181385945 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.922217632665706, + "std_abs_err": 45.65332550123571 + }, + "P4_ST_LD": { + "mean_abs_err": 0.09269243955748152, + "std_abs_err": 0.7481293959848188 + }, + "P4_ST_PO": { + "mean_abs_err": 0.10075867634122915, + "std_abs_err": 0.8130220041742859 + }, + "P4_ST_PT01": { + "mean_abs_err": 3.1558859857395873, + "std_abs_err": 2.8951415011737893 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.7717155076570634, + "std_abs_err": 0.8307807876826985 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.6595583333333334, + "P1_B2016": 0.04343750000000002, + "P1_B3004": 0.671875, + "P1_B3005": 0.5293, + "P1_B4002": 0.7923208333333334, + "P1_B4005": 0.3062625, + "P1_B400B": 0.4596875, + "P1_B4022": 0.5756333333333334, + "P1_FCV02Z": 0.7746, + "P1_FCV03D": 0.45162500000000005, + "P1_FCV03Z": 0.40985833333333327, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4726416666666667, + "P1_FT02": 0.4587083333333334, + "P1_FT02Z": 0.4029958333333334, + "P1_FT03": 0.5241, + "P1_FT03Z": 0.46950833333333336, + "P1_LCV01D": 0.3859916666666667, + "P1_LCV01Z": 0.397625, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5304583333333333, + "P1_PCV01Z": 0.5184416666666667, + "P1_PCV02Z": 0.6588541666666666, + "P1_PIT01": 0.15733333333333333, + "P1_PIT02": 0.61435, + "P1_TIT01": 0.09973333333333334, + "P1_TIT02": 0.10090833333333338, + "P2_24Vdc": 0.13548333333333334, + "P2_CO_rpm": 0.09332916666666669, + "P2_HILout": 0.08242499999999997, + "P2_MSD": 0.0, + "P2_SIT01": 0.4626833333333333, + "P2_SIT02": 0.5013166666666666, + "P2_VT01": 0.12326250000000005, + "P2_VXT02": 0.28197916666666667, + "P2_VXT03": 0.43657916666666663, + "P2_VYT02": 0.2505666666666666, + "P2_VYT03": 0.2697583333333333, + "P3_FIT01": 0.18958333333333338, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.08329166666666671, + "P3_LIT01": 0.09349166666666664, + "P3_PIT01": 0.4136708333333333, + "P4_HT_FD": 0.11226666666666663, + "P4_HT_LD": 0.06742500000000001, + "P4_HT_PO": 0.05970416666666667, + "P4_LD": 0.031079166666666658, + "P4_ST_FD": 0.03345833333333331, + "P4_ST_GOV": 0.03512083333333338, + "P4_ST_LD": 0.032354166666666684, + "P4_ST_PO": 0.03597499999999998, + "P4_ST_PT01": 0.16995416666666663, + "P4_ST_TT01": 0.08401666666666663 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.02845, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0011099999999999999, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.011969999999999814, + "q25_diff": 0.010260000000000158, + "q50_diff": 0.005159999999999831, + "q75_diff": 0.014349999999999863, + "q95_diff": 0.029469999999999885 + }, + "P1_B3004": { + "q05_diff": 14.80313000000001, + "q25_diff": 1.6197799999999916, + "q50_diff": 6.726189999999974, + "q75_diff": 8.508819999999957, + "q95_diff": 37.86444 + }, + "P1_B3005": { + "q05_diff": 40.40789999999993, + "q25_diff": 8.045049999999947, + "q50_diff": 89.51202, + "q75_diff": 104.97393, + "q95_diff": 4.4593499999998585 + }, + "P1_B4002": { + "q05_diff": 0.4564699999999995, + "q25_diff": 1.6555000000000035, + "q50_diff": 1.6555000000000035, + "q75_diff": 0.9287000000000063, + "q95_diff": 0.16969000000000278 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 81.96193, + "q50_diff": 0.04000000000000625, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2480.00824, + "q50_diff": 252.20629999999983, + "q75_diff": 9.545890000000327, + "q95_diff": 7.159419999999955 + }, + "P1_B4022": { + "q05_diff": 0.38222999999999985, + "q25_diff": 1.3479000000000028, + "q50_diff": 1.4273799999999994, + "q75_diff": 0.9127399999999994, + "q95_diff": 0.5270299999999963 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.05340999999999996, + "q75_diff": 38.169869999999996, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 1.3654200000000003, + "q25_diff": 0.2554099999999977, + "q50_diff": 8.60333, + "q75_diff": 10.84863, + "q95_diff": 0.824300000000008 + }, + "P1_FCV03Z": { + "q05_diff": 1.7547599999999974, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.148199999999996, + "q75_diff": 10.864249999999998, + "q95_diff": 0.9307899999999876 + }, + "P1_FT01": { + "q05_diff": 7.9056500000000085, + "q25_diff": 8.615820000000014, + "q50_diff": 32.61562999999998, + "q75_diff": 37.574780000000004, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 44.013919999999985, + "q25_diff": 17.829899999999952, + "q50_diff": 74.79840000000002, + "q75_diff": 84.80078000000003, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.2434799999999, + "q50_diff": 258.05504999999994, + "q75_diff": 13.160759999999982, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 1.9862300000000026, + "q25_diff": 2300.54303, + "q50_diff": 14.318849999999657, + "q75_diff": 7.299800000000232, + "q95_diff": 6.317140000000109 + }, + "P1_FT03": { + "q05_diff": 36.13194999999999, + "q25_diff": 1.5291600000000187, + "q50_diff": 51.52633000000003, + "q75_diff": 59.53728000000001, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 23.41980000000001, + "q25_diff": 2.9805299999999306, + "q50_diff": 84.98266000000012, + "q75_diff": 100.6383699999999, + "q95_diff": 3.5079399999999623 + }, + "P1_LCV01D": { + "q05_diff": 0.17497000000000007, + "q25_diff": 0.29953000000000074, + "q50_diff": 4.171619999999999, + "q75_diff": 3.162939999999999, + "q95_diff": 0.7125800000000009 + }, + "P1_LCV01Z": { + "q05_diff": 0.0, + "q25_diff": 0.5874600000000001, + "q50_diff": 4.638670000000001, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.9536700000000007 + }, + "P1_LIT01": { + "q05_diff": 14.301179999999988, + "q25_diff": 0.2044700000000148, + "q50_diff": 6.23626999999999, + "q75_diff": 8.229829999999993, + "q95_diff": 34.29950000000002 + }, + "P1_PCV01D": { + "q05_diff": 0.18625000000000114, + "q25_diff": 2.7490099999999984, + "q50_diff": 18.274499999999996, + "q75_diff": 43.266600000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.22888000000000375, + "q25_diff": 2.0370500000000007, + "q50_diff": 16.029360000000004, + "q75_diff": 43.33495, + "q95_diff": 45.1493 + }, + "P1_PCV02Z": { + "q05_diff": 0.2288800000000002, + "q25_diff": 0.2212499999999995, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_PIT01": { + "q05_diff": 0.029750000000000165, + "q25_diff": 0.011450000000000182, + "q50_diff": 0.009149999999999991, + "q75_diff": 0.022119999999999918, + "q95_diff": 0.035090000000000066 + }, + "P1_PIT02": { + "q05_diff": 0.028989999999999988, + "q25_diff": 0.09993999999999997, + "q50_diff": 0.10070999999999997, + "q75_diff": 0.0015300000000000313, + "q95_diff": 0.0022900000000000142 + }, + "P1_TIT01": { + "q05_diff": 0.045769999999997424, + "q25_diff": 0.07629999999999626, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.24206000000000216, + "q95_diff": 0.09154999999999802 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.030520000000002767, + "q50_diff": 0.09154999999999802, + "q75_diff": 0.30516999999999683, + "q95_diff": 0.4297200000000032 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.00013999999999825263, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.00010999999999938836 + }, + "P2_CO_rpm": { + "q05_diff": 11.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 2.0, + "q95_diff": 1.0 + }, + "P2_HILout": { + "q05_diff": 0.3295900000000529, + "q25_diff": 1.6540499999999838, + "q50_diff": 0.31737999999995736, + "q75_diff": 1.7334000000000742, + "q95_diff": 2.2399900000000343 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 1.0, + "q25_diff": 3.0, + "q50_diff": 5.0, + "q75_diff": 6.0, + "q95_diff": 6.0 + }, + "P2_SIT02": { + "q05_diff": 2.650700000000029, + "q25_diff": 3.569340000000011, + "q50_diff": 5.613770000000045, + "q75_diff": 6.357240000000047, + "q95_diff": 6.842530000000011 + }, + "P2_VT01": { + "q05_diff": 0.008340000000000458, + "q25_diff": 0.015269999999999229, + "q50_diff": 0.013980000000000103, + "q75_diff": 0.01343999999999923, + "q95_diff": 0.018990000000000506 + }, + "P2_VXT02": { + "q05_diff": 0.13161000000000023, + "q25_diff": 0.29940999999999995, + "q50_diff": 0.28532, + "q75_diff": 0.2669400000000004, + "q95_diff": 0.45862999999999987 + }, + "P2_VXT03": { + "q05_diff": 0.09976000000000007, + "q25_diff": 0.26910999999999996, + "q50_diff": 0.2972699999999999, + "q75_diff": 0.3267199999999999, + "q95_diff": 0.60987 + }, + "P2_VYT02": { + "q05_diff": 0.09552000000000005, + "q25_diff": 0.2620300000000002, + "q50_diff": 0.22682000000000002, + "q75_diff": 0.23567999999999945, + "q95_diff": 0.3888600000000002 + }, + "P2_VYT03": { + "q05_diff": 0.056390000000000384, + "q25_diff": 0.1858500000000003, + "q50_diff": 0.17562999999999995, + "q75_diff": 0.18247999999999998, + "q95_diff": 0.35470999999999986 + }, + "P3_FIT01": { + "q05_diff": 1.0, + "q25_diff": 0.0, + "q50_diff": 72.0, + "q75_diff": 2010.0, + "q95_diff": 63.0 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 88.22606999999994, + "q75_diff": 131.70507999999973, + "q95_diff": 10.988279999999577 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 256.0, + "q50_diff": 3227.02344, + "q75_diff": 192.0, + "q95_diff": 31.28319999999985 + }, + "P3_LIT01": { + "q05_diff": 1131.2470700000003, + "q25_diff": 28.58984000000055, + "q50_diff": 649.9677699999993, + "q75_diff": 1146.6367199999986, + "q95_diff": 1.662110000001121 + }, + "P3_PIT01": { + "q05_diff": 4.0, + "q25_diff": 5.0, + "q50_diff": 0.0, + "q75_diff": 928.0, + "q95_diff": 3524.0 + }, + "P4_HT_FD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 7e-05, + "q95_diff": 0.0 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 27.14122, + "q75_diff": 0.13020999999999106, + "q95_diff": 0.21701000000000192 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 0.9223299999999999, + "q50_diff": 8.89752, + "q75_diff": 0.054199999999994475, + "q95_diff": 0.6510700000000043 + }, + "P4_LD": { + "q05_diff": 0.5058300000000031, + "q25_diff": 3.0157100000000128, + "q50_diff": 3.449520000000007, + "q75_diff": 0.16302000000001726, + "q95_diff": 4.291780000000017 + }, + "P4_ST_FD": { + "q05_diff": 4.99999999999997e-05, + "q25_diff": 0.00013999999999999993, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.00014999999999999996 + }, + "P4_ST_GOV": { + "q05_diff": 125.97559000000001, + "q25_diff": 38.71679999999833, + "q50_diff": 30.52147999999943, + "q75_diff": 105.58594000000085, + "q95_diff": 402.84179999999833 + }, + "P4_ST_LD": { + "q05_diff": 0.5073900000000435, + "q25_diff": 0.40126999999995405, + "q50_diff": 0.4448600000000056, + "q75_diff": 2.0440699999999765, + "q95_diff": 7.88191999999998 + }, + "P4_ST_PO": { + "q05_diff": 2.2981300000000147, + "q25_diff": 0.7247299999999655, + "q50_diff": 0.6087400000000116, + "q75_diff": 2.2022400000000175, + "q95_diff": 7.818750000000023 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 20.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 30.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 7.0, + "q50_diff": 1.7675799999997253, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0624831607922858, + "P1_B2016": 0.09372153962205165, + "P1_B3004": 0.9420514424699565, + "P1_B3005": 0.9755864990985265, + "P1_B4002": 0.9939428775878422, + "P1_B4005": 1.0901635813805555, + "P1_B400B": 0.4439571613343968, + "P1_B4022": 1.0728287547128275, + "P1_FCV02Z": 1.0856923346443097, + "P1_FCV03D": 1.0287616800381814, + "P1_FCV03Z": 1.01920928123416, + "P1_FT01": 0.6609997822834346, + "P1_FT01Z": 0.3575180502180102, + "P1_FT02": 0.6690460556684432, + "P1_FT02Z": 1.0572315062387077, + "P1_FT03": 0.3022907751961552, + "P1_FT03Z": 0.9861265787637634, + "P1_LCV01D": 0.9922897795117445, + "P1_LCV01Z": 0.9035160861736835, + "P1_LIT01": 0.14869229177703291, + "P1_PCV01D": 0.9847169107083952, + "P1_PCV01Z": 1.1483081333334428, + "P1_PCV02Z": 0.5764046458797007, + "P1_PIT01": 0.2884406396378245, + "P1_PIT02": 0.16563474008732848, + "P1_TIT01": 0.043653145904043456, + "P1_TIT02": 0.08140759059784364, + "P2_24Vdc": 0.039965792976989475, + "P2_CO_rpm": 0.41590072608765727, + "P2_HILout": 0.24706059027478758, + "P2_MSD": 0.0, + "P2_SIT01": 0.49280073603354146, + "P2_SIT02": 0.5608912244785349, + "P2_VT01": 0.14955850806894322, + "P2_VXT02": 0.1622295159815592, + "P2_VXT03": 0.19162944594539777, + "P2_VYT02": 0.15176778852528572, + "P2_VYT03": 0.1718451687373067, + "P3_FIT01": 0.16161011867720687, + "P3_LCP01D": 0.14050969105480315, + "P3_LCV01D": 0.347878557710531, + "P3_LIT01": 0.07579325279870874, + "P3_PIT01": 0.7237460833164835, + "P4_HT_FD": 0.05083342513502456, + "P4_HT_LD": 1.052071148480924, + "P4_HT_PO": 1.0918064404320895, + "P4_LD": 0.06981743984686695, + "P4_ST_FD": 0.2809423474777602, + "P4_ST_GOV": 0.11942940208112951, + "P4_ST_LD": 0.15564231905790638, + "P4_ST_PO": 0.1019794048923851, + "P4_ST_PT01": 0.8083994035521874, + "P4_ST_TT01": 0.05517927948168333 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.5505791762707029, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_file_condition__seed1337/comprehensive_eval.json b/example/results/ablations/runs/no_file_condition__seed1337/comprehensive_eval.json new file mode 100644 index 0000000..94e9989 --- /dev/null +++ b/example/results/ablations/runs/no_file_condition__seed1337/comprehensive_eval.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_file_condition__seed1337/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.4049190251572327, + "avg_jsd": 0.0327003560472195, + "avg_lag1_diff": 0.23744482453266894 + }, + "two_sample": { + "continuous_mmd_rbf": 0.6075121164321899, + "continuous_mmd_gamma": 1.6257409663734532e-05, + "histogram_mmd_rbf": 0.6075193881988525, + "histogram_mmd_gamma": 1.625650530095427e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 166.45201110839844, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 3.016726244982856, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.262408971786499, + "corr_frobenius": 20.80748748779297, + "lag1_corr_mean_abs_diff": 0.2604692876338959, + "lag1_corr_frobenius": 20.88396453857422, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.38173002004623413, + "corr_frobenius": 13.880753517150879, + "lag1_corr_mean_abs_diff": 0.39038363099098206, + "lag1_corr_frobenius": 14.076647758483887 + }, + "P2": { + "corr_mean_abs_diff": 0.10715697705745697, + "corr_frobenius": 2.10937762260437, + "lag1_corr_mean_abs_diff": 0.13567495346069336, + "lag1_corr_frobenius": 2.75299072265625 + }, + "P3": { + "corr_mean_abs_diff": 0.4653128683567047, + "corr_frobenius": 2.981515407562256, + "lag1_corr_mean_abs_diff": 0.4981975555419922, + "lag1_corr_frobenius": 3.027890682220459 + }, + "P4": { + "corr_mean_abs_diff": 0.2012164294719696, + "corr_frobenius": 2.9119718074798584, + "lag1_corr_mean_abs_diff": 0.2096780389547348, + "lag1_corr_frobenius": 2.9789016246795654 + } + } + }, + "frequency": { + "avg_psd_l1": 0.013894757255911827, + "avg_psd_cosine": 0.2153413106332484, + "avg_low_high_ratio_abs_diff": 9433973.003211735 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 15.0, + "mean_dwell": 24.0, + "median_dwell": 2.5, + "mean_step": 68.42660395304362, + "median_step": 1.9043693542480469 + }, + "P1_B2004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B3004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.01115664489586733, + "median_step": 0.004810333251953125 + }, + "P1_B3005": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22998.14285714286, + "mean_abs_diff_mean_dwell": 14201.625685203559, + "mean_abs_diff_median_dwell": 14207.642857142857, + "mean_abs_diff_mean_step": 33.693117622162895, + "mean_abs_diff_median_step": 0.831573486328125 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6041666666666666, + "change_rate": 0.5848563968668408, + "step_median": 20.943649291992188 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.01520792643229163, + "mean_abs_diff_change_rate": 0.12403627947469609, + "mean_abs_diff_step_median": 20.911628991365433 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.0026041666666666665, + "top1_mass": 1.0, + "top3_mass": 1.0, + "median_dwell": 384.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.9895833333333334, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9192708333333334, + "top1_mass": 0.08333333333333333, + "top3_mass": 0.08854166666666667, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.07552083333333333, + "top1_mass": 0.16666666666666666, + "top3_mass": 0.4739583333333333, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.9973958333333334, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.010416666666666666, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.5286458333333334, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.96875, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.015625, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.18489583333333334, + "top1_mass": 0.19270833333333334, + "top3_mass": 0.3515625, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.551507314046224, + "mean_abs_diff_top1_mass": 0.09950892130533853, + "mean_abs_diff_top3_mass": 0.06765111287434895, + "mean_abs_diff_median_dwell": 47.75 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17333999276161194, + "q50": 0.27557000517845154, + "q95": 1.8291699886322021, + "tail_ratio": 15.197102560439848 + }, + "P2_SIT02": { + "q05": 774.9176025390625, + "q50": 784.55615234375, + "q95": 791.7053833007812, + "tail_ratio": 0.7417330513304373 + }, + "P1_FT03": { + "q05": 208.92420959472656, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.1925292233340415 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.576893558104833, + "mean_abs_diff_q50": 2.23438752690951, + "mean_abs_diff_q95": 3.0419100522994995, + "mean_abs_diff_tail_ratio": 2.1637767804458545 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 354.0, + "mean_dwell": 1.0816901408450703, + "median_dwell": 1.0, + "mean_step": 4.456174688824152, + "median_step": 1.2019882202148438 + }, + "P1_FT02Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 921.2004766087783, + "median_step": 768.2478334903717 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89295.5, + "mean_abs_diff_mean_dwell": 0.05614842203570802, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 455.1772250732858, + "mean_abs_diff_median_step": 381.458004117012 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 25.825477600097656, + "std": 33.53642654418945, + "lag1": 0.9658128023147583 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.0035480197984725237, + "lag1": -0.07761066406965256 + }, + "P2_HILout": { + "mean": 711.482421875, + "std": 9.428905487060547, + "lag1": 0.03969178721308708 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 3.008173624674479, + "mean_abs_diff_std": 1.1354477227044601, + "mean_abs_diff_lag1": 0.12880567957957587 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9855486154556274, + "mae": 0.7909917831420898 + }, + "real_plus_synthetic": { + "rmse": 0.5445631742477417, + "mae": 0.3514666259288788 + }, + "rmse_ratio_synth_to_real": 1.765106098975039 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8293599257884972, + "auprc": 0.5891939993813046, + "threshold": 0.013085918501019478, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6756756756756758, + "best_f1_threshold": 1.4744409322738647 + }, + "real_plus_synthetic": { + "auroc": 0.8753246753246753, + "auprc": 0.640266746683905, + "threshold": 0.3295350670814514, + "f1": 0.26839826839826836, + "precision": 0.1657754010695187, + "recall": 0.7045454545454546, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.542262852191925 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_file_condition__seed1337/comprehensive_eval_post.json b/example/results/ablations/runs/no_file_condition__seed1337/comprehensive_eval_post.json new file mode 100644 index 0000000..16a2f55 --- /dev/null +++ b/example/results/ablations/runs/no_file_condition__seed1337/comprehensive_eval_post.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_file_condition__seed1337/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.1552201509475708, + "continuous_mmd_gamma": 4.357279563141874e-05, + "histogram_mmd_rbf": 0.1553114652633667, + "histogram_mmd_gamma": 4.356769006489884e-05, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 97.8538589477539, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 1.7734739429990898, + "one_nn_two_sample_accuracy": 0.9979674796747967 + }, + "coupling": { + "corr_mean_abs_diff": 0.2079719752073288, + "corr_frobenius": 17.314613342285156, + "lag1_corr_mean_abs_diff": 0.21452701091766357, + "lag1_corr_frobenius": 17.90935516357422, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.35661715269088745, + "corr_frobenius": 12.46723461151123, + "lag1_corr_mean_abs_diff": 0.3795819580554962, + "lag1_corr_frobenius": 13.13932991027832 + }, + "P2": { + "corr_mean_abs_diff": 0.07869446277618408, + "corr_frobenius": 1.44179105758667, + "lag1_corr_mean_abs_diff": 0.1392807811498642, + "lag1_corr_frobenius": 2.752474784851074 + }, + "P3": { + "corr_mean_abs_diff": 0.4653128683567047, + "corr_frobenius": 2.981515407562256, + "lag1_corr_mean_abs_diff": 0.4981975555419922, + "lag1_corr_frobenius": 3.027890682220459 + }, + "P4": { + "corr_mean_abs_diff": 0.2585130035877228, + "corr_frobenius": 3.6911299228668213, + "lag1_corr_mean_abs_diff": 0.27941733598709106, + "lag1_corr_frobenius": 3.9866018295288086 + } + } + }, + "frequency": { + "avg_psd_l1": 0.018553154543042183, + "avg_psd_cosine": 0.36147105553240144, + "avg_low_high_ratio_abs_diff": 17.616489490897184 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 308.0, + "mean_dwell": 1.2427184466019416, + "median_dwell": 1.0, + "mean_step": 0.8413416007896523, + "median_step": 0.7274894714355469 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 347.0, + "mean_dwell": 1.103448275862069, + "median_dwell": 1.0, + "mean_step": 41.3141472878254, + "median_step": 54.116028785705566 + }, + "P1_B2004": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 0.014068839402653235, + "median_step": 0.0017450004816055298 + }, + "P1_B3004": { + "num_changes": 370.0, + "mean_dwell": 1.0350404312668464, + "median_dwell": 1.0, + "mean_step": 14.77256890374261, + "median_step": 10.804779052734375 + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.8157603410765646, + "median_step": 0.5944099426269531 + }, + "P1_B3005": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 64.28072175926926, + "median_step": 41.228179931640625 + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22997.85714285714, + "mean_abs_diff_mean_dwell": 14417.281142645239, + "mean_abs_diff_median_dwell": 13988.57142857143, + "mean_abs_diff_mean_step": 19.03385039702909, + "mean_abs_diff_median_step": 16.840154950817425 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6822916666666666, + "change_rate": 0.7702349869451697, + "step_median": 68.81373023986816 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.09333292643229163, + "mean_abs_diff_change_rate": 0.309414869553025, + "mean_abs_diff_step_median": 68.78170993924141 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.015625, + "top1_mass": 0.40625, + "top3_mass": 0.7760416666666666, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8776041666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.059895833333333336, + "top3_mass": 0.0703125, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.18229166666666666, + "top3_mass": 0.3697916666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.8463541666666666, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4817708333333333, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.7864583333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3515625, + "top1_mass": 0.036458333333333336, + "top3_mass": 0.09635416666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5303484598795573, + "mean_abs_diff_top1_mass": 0.03112411499023438, + "mean_abs_diff_top3_mass": 0.05629857381184896, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17333999276161194, + "q50": 0.27557000517845154, + "q95": 1.8291699886322021, + "tail_ratio": 15.197102560439848 + }, + "P2_SIT02": { + "q05": 774.9176025390625, + "q50": 784.55615234375, + "q95": 791.7053833007812, + "tail_ratio": 0.7417330513304373 + }, + "P1_FT03": { + "q05": 208.92420959472656, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.1925292233340415 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.576893558104833, + "mean_abs_diff_q50": 2.23438752690951, + "mean_abs_diff_q95": 3.0419100522994995, + "mean_abs_diff_tail_ratio": 2.1637767804458545 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 64.74391126130756, + "median_step": 62.410614013671875 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 1312.3134692358592, + "median_step": 1284.6439208984375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.09039718638964156, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 680.877589673068, + "mean_abs_diff_median_step": 670.2603607177734 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 35.254512786865234, + "std": 31.650897979736328, + "lag1": -0.0927804708480835 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.003297113813459873, + "lag1": 0.06643600016832352 + }, + "P2_HILout": { + "mean": 711.4987182617188, + "std": 7.856828212738037, + "lag1": -0.013758387416601181 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.4565467834472656, + "mean_abs_diff_std": 0.2373291968833655, + "mean_abs_diff_lag1": 0.4579590285817782 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.972450315952301, + "mae": 0.7795692682266235 + }, + "real_plus_synthetic": { + "rmse": 0.5614052414894104, + "mae": 0.35388970375061035 + }, + "rmse_ratio_synth_to_real": 1.7416471970224094 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8280148423005567, + "auprc": 0.5923964717575256, + "threshold": 0.009861066937446594, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6756756756756758, + "best_f1_threshold": 1.4294195175170898 + }, + "real_plus_synthetic": { + "auroc": 0.8716372912801484, + "auprc": 0.6414936908045138, + "threshold": 0.29818135499954224, + "f1": 0.25098039215686274, + "precision": 0.15165876777251186, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.5320196151733398 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_file_condition__seed1337/eval.json b/example/results/ablations/runs/no_file_condition__seed1337/eval.json new file mode 100644 index 0000000..f541242 --- /dev/null +++ b/example/results/ablations/runs/no_file_condition__seed1337/eval.json @@ -0,0 +1,974 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.11384, + "std": 0.0 + }, + "P1_B2016": { + "mean": 1.391534401041668, + "std": 0.13946106107922276 + }, + "P1_B3004": { + "mean": 396.00211, + "std": 0.0 + }, + "P1_B3005": { + "mean": 1015.43976, + "std": 0.0 + }, + "P1_B4002": { + "mean": 33.54375, + "std": 0.0 + }, + "P1_B4005": { + "mean": 60.49637799479164, + "std": 42.261442188457764 + }, + "P1_B400B": { + "mean": 1777.596925755208, + "std": 1195.6797141995014 + }, + "P1_B4022": { + "mean": 36.881583541666664, + "std": 0.14105102171131376 + }, + "P1_FCV02Z": { + "mean": 20.035361953125005, + "std": 35.32475070063646 + }, + "P1_FCV03D": { + "mean": 58.57116992187503, + "std": 6.64132219713204 + }, + "P1_FCV03Z": { + "mean": 59.24646770833335, + "std": 6.823641329634568 + }, + "P1_FT01": { + "mean": 194.69574812500008, + "std": 32.18984962125104 + }, + "P1_FT01Z": { + "mean": 875.3945892447917, + "std": 99.56005373645125 + }, + "P1_FT02": { + "mean": 1132.49033296875, + "std": 890.6858387444685 + }, + "P1_FT02Z": { + "mean": 1132.49033296875, + "std": 890.6858387444685 + }, + "P1_FT03": { + "mean": 282.27597710937533, + "std": 33.67135723115452 + }, + "P1_FT03Z": { + "mean": 282.27597710937533, + "std": 33.67135723115452 + }, + "P1_LCV01D": { + "mean": 12.168015208333324, + "std": 3.0474337416476507 + }, + "P1_LCV01Z": { + "mean": 11.825347604166666, + "std": 3.1912465232585707 + }, + "P1_LIT01": { + "mean": 404.0009294010416, + "std": 15.70528322380496 + }, + "P1_PCV01D": { + "mean": 67.44933604166657, + "std": 23.853524895759502 + }, + "P1_PCV01Z": { + "mean": 68.38966971354166, + "std": 23.945880723195838 + }, + "P1_PCV02Z": { + "mean": 11.76605, + "std": 0.0 + }, + "P1_PIT01": { + "mean": 1.3749083333333338, + "std": 0.059795921368924275 + }, + "P1_PIT02": { + "mean": 0.47586984374999997, + "std": 0.510092890346671 + }, + "P1_TIT01": { + "mean": 35.65237570312499, + "std": 0.5899993262509078 + }, + "P1_TIT02": { + "mean": 36.48672705729164, + "std": 1.220891267055021 + }, + "P2_24Vdc": { + "mean": 28.028605651041673, + "std": 0.0035479543759593026 + }, + "P2_CO_rpm": { + "mean": 54102.28249101564, + "std": 23.951768712470482 + }, + "P2_HILout": { + "mean": 711.4824051562496, + "std": 9.42890457519887 + }, + "P2_MSD": { + "mean": 758.4425, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.2530208333329, + "std": 5.357406258330356 + }, + "P2_SIT02": { + "mean": 783.9234037239586, + "std": 5.839501711639723 + }, + "P2_VT01": { + "mean": 11.923781276041664, + "std": 0.06898546777898959 + }, + "P2_VXT02": { + "mean": -3.299332369791668, + "std": 0.47724095306989195 + }, + "P2_VXT03": { + "mean": -1.3056848958333336, + "std": 0.35729730024065115 + }, + "P2_VYT02": { + "mean": 3.9808586718749996, + "std": 0.5057069185651452 + }, + "P2_VYT03": { + "mean": 6.233392708333337, + "std": 0.38918658337931306 + }, + "P3_FIT01": { + "mean": 959.5200457291662, + "std": 1756.2267796064482 + }, + "P3_LCP01D": { + "mean": 4590.908203072916, + "std": 5143.586765092662 + }, + "P3_LCV01D": { + "mean": 6167.148197187503, + "std": 7091.614281472208 + }, + "P3_LIT01": { + "mean": 13352.648086588542, + "std": 4370.825816197887 + }, + "P3_PIT01": { + "mean": 250.3358968229168, + "std": 490.47959019410433 + }, + "P4_HT_FD": { + "mean": -0.002084687499999999, + "std": 0.010185916807414626 + }, + "P4_HT_LD": { + "mean": -115.69799739583333, + "std": 35.27327640165136 + }, + "P4_HT_PO": { + "mean": 25.825478098958335, + "std": 33.53642937999895 + }, + "P4_LD": { + "mean": 366.1226154687502, + "std": 61.03142784766532 + }, + "P4_ST_FD": { + "mean": -5.903645833333325e-05, + "std": 0.004046068698763828 + }, + "P4_ST_GOV": { + "mean": 17918.506001718742, + "std": 1896.6960206854487 + }, + "P4_ST_LD": { + "mean": 332.09312765625, + "std": 38.5901339955848 + }, + "P4_ST_PO": { + "mean": 332.3664021614585, + "std": 35.26147496196201 + }, + "P4_ST_PT01": { + "mean": 10039.861785911464, + "std": 15.884781682619622 + }, + "P4_ST_TT01": { + "mean": 27608.10104877603, + "std": 24.783333022508646 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.020272347909024324, + "std_abs_err": 0.017998437082092405 + }, + "P1_B2016": { + "mean_abs_err": 0.00026807637690096087, + "std_abs_err": 0.0036874986026596834 + }, + "P1_B3004": { + "mean_abs_err": 7.383953824526145, + "std_abs_err": 15.41814958804181 + }, + "P1_B3005": { + "mean_abs_err": 45.609026528629784, + "std_abs_err": 61.18331651100028 + }, + "P1_B4002": { + "mean_abs_err": 1.2268236801201198, + "std_abs_err": 0.5786734991990778 + }, + "P1_B4005": { + "mean_abs_err": 0.02872902140628497, + "std_abs_err": 0.0801447407480751 + }, + "P1_B400B": { + "mean_abs_err": 0.8789444227600143, + "std_abs_err": 2.3274412804287294 + }, + "P1_B4022": { + "mean_abs_err": 0.8204088942641405, + "std_abs_err": 0.5303998331235046 + }, + "P1_FCV02Z": { + "mean_abs_err": 4.915572947347272, + "std_abs_err": 5.113734738621723 + }, + "P1_FCV03D": { + "mean_abs_err": 0.0033064710282033616, + "std_abs_err": 0.04182647842328535 + }, + "P1_FCV03Z": { + "mean_abs_err": 0.05257147185648847, + "std_abs_err": 0.10883736530296595 + }, + "P1_FT01": { + "mean_abs_err": 0.0279062312801841, + "std_abs_err": 4.841867782606716 + }, + "P1_FT01Z": { + "mean_abs_err": 3.6550922593607993, + "std_abs_err": 23.794350773133004 + }, + "P1_FT02": { + "mean_abs_err": 0.3278202658761984, + "std_abs_err": 1.4896748930558488 + }, + "P1_FT02Z": { + "mean_abs_err": 645.9881136398988, + "std_abs_err": 302.6595170266811 + }, + "P1_FT03": { + "mean_abs_err": 0.3629931668535846, + "std_abs_err": 0.19714411985901847 + }, + "P1_FT03Z": { + "mean_abs_err": 778.7660612391462, + "std_abs_err": 27.454131632462804 + }, + "P1_LCV01D": { + "mean_abs_err": 0.013682546558188236, + "std_abs_err": 0.13281878861752938 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.1407264251712519, + "std_abs_err": 0.2277770259132592 + }, + "P1_LIT01": { + "mean_abs_err": 0.11394276196466535, + "std_abs_err": 0.07338517543098888 + }, + "P1_PCV01D": { + "mean_abs_err": 0.016237694024809457, + "std_abs_err": 0.0837581948539281 + }, + "P1_PCV01Z": { + "mean_abs_err": 0.018482083360922275, + "std_abs_err": 0.08128091253219338 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.07670294660492694, + "std_abs_err": 0.09684446546464737 + }, + "P1_PIT01": { + "mean_abs_err": 0.002572887420058123, + "std_abs_err": 0.0016366380009578269 + }, + "P1_PIT02": { + "mean_abs_err": 0.011287246591516753, + "std_abs_err": 0.01138125229179121 + }, + "P1_TIT01": { + "mean_abs_err": 0.028832539308226046, + "std_abs_err": 0.006718699064958811 + }, + "P1_TIT02": { + "mean_abs_err": 0.03825162104489266, + "std_abs_err": 0.0008674589692421186 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00010405158599624542, + "std_abs_err": 0.00020548826155238759 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.4277068670344306, + "std_abs_err": 1.496036937682117 + }, + "P2_HILout": { + "mean_abs_err": 0.019595886153410902, + "std_abs_err": 0.9741786979427065 + }, + "P2_MSD": { + "mean_abs_err": 4.750739999999951, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.650896580110043, + "std_abs_err": 0.15849089788613124 + }, + "P2_SIT02": { + "mean_abs_err": 0.019665841412347618, + "std_abs_err": 0.6489219548267569 + }, + "P2_VT01": { + "mean_abs_err": 3.436996846950535e-05, + "std_abs_err": 0.0004717857069875264 + }, + "P2_VXT02": { + "mean_abs_err": 0.00084610259832818, + "std_abs_err": 0.006208026798671662 + }, + "P2_VXT03": { + "mean_abs_err": 0.0012083908313442748, + "std_abs_err": 0.007669921076997488 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006101781179372878, + "std_abs_err": 0.005796465572357301 + }, + "P2_VYT03": { + "mean_abs_err": 0.0010124044645474228, + "std_abs_err": 0.009383621408221465 + }, + "P3_FIT01": { + "mean_abs_err": 191.38013037701694, + "std_abs_err": 23.49791818382664 + }, + "P3_LCP01D": { + "mean_abs_err": 6.1810548323419425, + "std_abs_err": 13.586767376783428 + }, + "P3_LCV01D": { + "mean_abs_err": 966.728519793739, + "std_abs_err": 251.68638565638685 + }, + "P3_LIT01": { + "mean_abs_err": 2.1964061054441117, + "std_abs_err": 16.65117926995117 + }, + "P3_PIT01": { + "mean_abs_err": 399.02200620041094, + "std_abs_err": 660.442340513091 + }, + "P4_HT_FD": { + "mean_abs_err": 0.001981499554648259, + "std_abs_err": 0.008153898493010526 + }, + "P4_HT_LD": { + "mean_abs_err": 149.69291447551893, + "std_abs_err": 1.8494828916530537 + }, + "P4_HT_PO": { + "mean_abs_err": 8.158329695798571, + "std_abs_err": 2.0955581626774737 + }, + "P4_LD": { + "mean_abs_err": 0.0061383254441125246, + "std_abs_err": 0.4156214782223131 + }, + "P4_ST_FD": { + "mean_abs_err": 2.992322845018443e-05, + "std_abs_err": 0.002403892440773799 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.544740471217665, + "std_abs_err": 44.552804697675356 + }, + "P4_ST_LD": { + "mean_abs_err": 0.09023048643229004, + "std_abs_err": 0.7369647039672529 + }, + "P4_ST_PO": { + "mean_abs_err": 0.0974308377997204, + "std_abs_err": 0.8105314595537081 + }, + "P4_ST_PT01": { + "mean_abs_err": 10.71337090761881, + "std_abs_err": 1.8999883426995083 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.9316429035025067, + "std_abs_err": 0.8660112179403185 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 1.0, + "P1_B2016": 0.04325000000000001, + "P1_B3004": 0.8108, + "P1_B3005": 0.8106, + "P1_B4002": 0.81055, + "P1_B4005": 0.3811333333333334, + "P1_B400B": 0.45285833333333336, + "P1_B4022": 0.5730333333333334, + "P1_FCV02Z": 0.7537666666666666, + "P1_FCV03D": 0.4879375, + "P1_FCV03Z": 0.4923916666666667, + "P1_FT01": 0.4746, + "P1_FT01Z": 0.4750083333333333, + "P1_FT02": 0.4531416666666666, + "P1_FT02Z": 0.8361000000000001, + "P1_FT03": 0.5241, + "P1_FT03Z": 1.0, + "P1_LCV01D": 0.4471333333333333, + "P1_LCV01Z": 0.45022916666666674, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5576625, + "P1_PCV01Z": 0.5588041666666668, + "P1_PCV02Z": 1.0, + "P1_PIT01": 0.16370833333333334, + "P1_PIT02": 0.3022, + "P1_TIT01": 0.10502083333333334, + "P1_TIT02": 0.09962083333333338, + "P2_24Vdc": 0.1226208333333334, + "P2_CO_rpm": 0.09815833333333329, + "P2_HILout": 0.08204583333333332, + "P2_MSD": 1.0, + "P2_SIT01": 0.48227083333333326, + "P2_SIT02": 0.5013166666666666, + "P2_VT01": 0.12296249999999997, + "P2_VXT02": 0.2815791666666666, + "P2_VXT03": 0.43657500000000005, + "P2_VYT02": 0.25022083333333334, + "P2_VYT03": 0.26955833333333334, + "P3_FIT01": 0.19270833333333337, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.5260416666666666, + "P3_LIT01": 0.09349166666666664, + "P3_PIT01": 0.339975, + "P4_HT_FD": 0.1423458333333334, + "P4_HT_LD": 0.9375, + "P4_HT_PO": 0.53685, + "P4_LD": 0.03121666666666667, + "P4_ST_FD": 0.023041666666666683, + "P4_ST_GOV": 0.03482083333333341, + "P4_ST_LD": 0.03254166666666658, + "P4_ST_PO": 0.035720833333333424, + "P4_ST_PT01": 0.24407916666666668, + "P4_ST_TT01": 0.07099583333333337 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.08333, + "q25_diff": 0.012630000000000002, + "q50_diff": 0.012630000000000002, + "q75_diff": 0.012630000000000002, + "q95_diff": 0.012630000000000002 + }, + "P1_B2016": { + "q05_diff": 0.011359999999999815, + "q25_diff": 0.010220000000000118, + "q50_diff": 0.005260000000000042, + "q75_diff": 0.014270000000000005, + "q95_diff": 0.030689999999999884 + }, + "P1_B3004": { + "q05_diff": 2.8398500000000126, + "q25_diff": 1.6357399999999984, + "q50_diff": 1.6357399999999984, + "q75_diff": 1.6357399999999984, + "q95_diff": 1.6357399999999984 + }, + "P1_B3005": { + "q05_diff": 13.44177000000002, + "q25_diff": 13.44177000000002, + "q50_diff": 13.44177000000002, + "q75_diff": 13.44177000000002, + "q95_diff": 99.67999000000009 + }, + "P1_B4002": { + "q05_diff": 1.5437500000000028, + "q25_diff": 0.11175000000000068, + "q50_diff": 0.11175000000000068, + "q75_diff": 0.11175000000000068, + "q95_diff": 0.11175000000000068 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 88.14547, + "q50_diff": 20.56854, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2478.8725, + "q50_diff": 192.26342999999997, + "q75_diff": 9.545890000000327, + "q95_diff": 7.019040000000132 + }, + "P1_B4022": { + "q05_diff": 1.2294699999999992, + "q25_diff": 0.1574500000000043, + "q50_diff": 0.4875299999999996, + "q75_diff": 0.5946499999999943, + "q95_diff": 0.6461800000000011 + }, + "P1_FCV02Z": { + "q05_diff": 0.0, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.06867000000000001, + "q75_diff": 22.67075, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 2.902799999999999, + "q25_diff": 0.0951999999999984, + "q50_diff": 8.54448, + "q75_diff": 11.160199999999996, + "q95_diff": 0.7484800000000007 + }, + "P1_FCV03Z": { + "q05_diff": 3.52478, + "q25_diff": 0.022889999999996746, + "q50_diff": 8.96454, + "q75_diff": 11.566150000000007, + "q95_diff": 0.5830500000000001 + }, + "P1_FT01": { + "q05_diff": 13.542220000000015, + "q25_diff": 8.20162000000002, + "q50_diff": 32.806329999999974, + "q75_diff": 37.95618999999999, + "q95_diff": 2.861050000000006 + }, + "P1_FT01Z": { + "q05_diff": 38.05517999999995, + "q25_diff": 19.59997999999996, + "q50_diff": 74.79840000000002, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.14705, + "q50_diff": 257.5476000000001, + "q75_diff": 12.970209999999952, + "q95_diff": 10.108890000000201 + }, + "P1_FT02Z": { + "q05_diff": 27.82217, + "q25_diff": 2761.08784, + "q50_diff": 1117.12195, + "q75_diff": 870.8322700000001, + "q95_diff": 865.8059100000003 + }, + "P1_FT03": { + "q05_diff": 36.18037000000001, + "q25_diff": 1.363020000000006, + "q50_diff": 51.52633000000003, + "q75_diff": 58.58354000000003, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 788.91423, + "q25_diff": 752.83569, + "q50_diff": 703.89301, + "q75_diff": 698.78882, + "q95_diff": 800.17028 + }, + "P1_LCV01D": { + "q05_diff": 0.10797999999999952, + "q25_diff": 1.210890000000001, + "q50_diff": 4.63538, + "q75_diff": 3.31161, + "q95_diff": 0.6148299999999995 + }, + "P1_LCV01Z": { + "q05_diff": 0.7019000000000002, + "q25_diff": 1.182549999999999, + "q50_diff": 4.9748, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.7107900000000011 + }, + "P1_LIT01": { + "q05_diff": 14.325409999999977, + "q25_diff": 1.04367000000002, + "q50_diff": 6.2459499999999935, + "q75_diff": 8.229829999999993, + "q95_diff": 34.29950000000002 + }, + "P1_PCV01D": { + "q05_diff": 1.0556199999999976, + "q25_diff": 3.71396, + "q50_diff": 22.159179999999992, + "q75_diff": 44.8556, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.959140000000005, + "q25_diff": 3.814699999999995, + "q50_diff": 22.73332, + "q75_diff": 45.46356000000001, + "q95_diff": 45.14375 + }, + "P1_PCV02Z": { + "q05_diff": 0.2441399999999998, + "q25_diff": 0.2441399999999998, + "q50_diff": 0.2517700000000005, + "q75_diff": 0.2517700000000005, + "q95_diff": 0.2517700000000005 + }, + "P1_PIT01": { + "q05_diff": 0.03493000000000013, + "q25_diff": 0.017550000000000177, + "q50_diff": 0.0045800000000000285, + "q75_diff": 0.024410000000000043, + "q95_diff": 0.030510000000000037 + }, + "P1_PIT02": { + "q05_diff": 0.03128, + "q25_diff": 0.06789999999999999, + "q50_diff": 0.0015299999999999758, + "q75_diff": 0.08316000000000001, + "q95_diff": 1.52573 + }, + "P1_TIT01": { + "q05_diff": 0.015270000000001005, + "q25_diff": 0.0457800000000006, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.27465999999999724, + "q95_diff": 0.045769999999997424 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.06367000000000189, + "q50_diff": 0.21363000000000198, + "q75_diff": 0.3044799999999981, + "q95_diff": 0.6713900000000024 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 2.999999999886427e-05, + "q50_diff": 7.000000000090267e-05, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.000920000000000698 + }, + "P2_CO_rpm": { + "q05_diff": 10.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 3.0, + "q95_diff": 0.0 + }, + "P2_HILout": { + "q05_diff": 0.5224699999999984, + "q25_diff": 2.0181199999999535, + "q50_diff": 0.08898999999996704, + "q75_diff": 1.5332100000000537, + "q95_diff": 1.1611400000000458 + }, + "P2_MSD": { + "q05_diff": 4.750739999999951, + "q25_diff": 4.750739999999951, + "q50_diff": 4.750739999999951, + "q75_diff": 4.750739999999951, + "q95_diff": 4.750739999999951 + }, + "P2_SIT01": { + "q05_diff": 1.0, + "q25_diff": 3.0, + "q50_diff": 5.0, + "q75_diff": 6.0, + "q95_diff": 5.0 + }, + "P2_SIT02": { + "q05_diff": 2.657410000000027, + "q25_diff": 3.569340000000011, + "q50_diff": 5.614020000000096, + "q75_diff": 6.363950000000045, + "q95_diff": 6.839899999999943 + }, + "P2_VT01": { + "q05_diff": 0.008340000000000458, + "q25_diff": 0.015240000000000364, + "q50_diff": 0.01422999999999952, + "q75_diff": 0.013299999999999201, + "q95_diff": 0.019059999999999633 + }, + "P2_VXT02": { + "q05_diff": 0.13016000000000005, + "q25_diff": 0.29947999999999997, + "q50_diff": 0.28572, + "q75_diff": 0.2670600000000003, + "q95_diff": 0.45709 + }, + "P2_VXT03": { + "q05_diff": 0.09953999999999996, + "q25_diff": 0.26919000000000004, + "q50_diff": 0.29715999999999987, + "q75_diff": 0.32733999999999996, + "q95_diff": 0.6096 + }, + "P2_VYT02": { + "q05_diff": 0.09771999999999981, + "q25_diff": 0.26203999999999983, + "q50_diff": 0.22734000000000032, + "q75_diff": 0.2356600000000002, + "q95_diff": 0.3893300000000002 + }, + "P2_VYT03": { + "q05_diff": 0.05701999999999963, + "q25_diff": 0.1859200000000003, + "q50_diff": 0.17567999999999984, + "q75_diff": 0.18238999999999983, + "q95_diff": 0.3547399999999996 + }, + "P3_FIT01": { + "q05_diff": 5.0, + "q25_diff": 2.0, + "q50_diff": 67.0, + "q75_diff": 2512.0, + "q95_diff": 17.22606999999971 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 90.70459000000005, + "q75_diff": 123.05469000000085, + "q95_diff": 12.901369999999588 + }, + "P3_LCV01D": { + "q05_diff": 16.0, + "q25_diff": 336.0, + "q50_diff": 9488.0, + "q75_diff": 976.0, + "q95_diff": 31.57227000000057 + }, + "P3_LIT01": { + "q05_diff": 1132.3847699999997, + "q25_diff": 26.040039999999863, + "q50_diff": 642.8085900000005, + "q75_diff": 1150.1914099999995, + "q95_diff": 4.4375 + }, + "P3_PIT01": { + "q05_diff": 4.0, + "q25_diff": 0.0, + "q50_diff": 1.0, + "q75_diff": 929.0, + "q95_diff": 2364.0 + }, + "P4_HT_FD": { + "q05_diff": 6.999999999999997e-05, + "q25_diff": 0.00014, + "q50_diff": 7.000000000000002e-05, + "q75_diff": 0.0, + "q95_diff": 0.00029 + }, + "P4_HT_LD": { + "q05_diff": 124.7862, + "q25_diff": 124.7862, + "q50_diff": 179.53157, + "q75_diff": 193.36982, + "q95_diff": 56.57957999999999 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 1.35638, + "q50_diff": 43.402800000000006, + "q75_diff": 1.844539999999995, + "q95_diff": 0.09221000000000856 + }, + "P4_LD": { + "q05_diff": 0.5106200000000172, + "q25_diff": 3.088860000000011, + "q50_diff": 3.4461600000000203, + "q75_diff": 0.07796999999999343, + "q95_diff": 4.251340000000027 + }, + "P4_ST_FD": { + "q05_diff": 1.0000000000000026e-05, + "q25_diff": 0.0, + "q50_diff": 7.999999999999999e-05, + "q75_diff": 0.0, + "q95_diff": 8.000000000000021e-05 + }, + "P4_ST_GOV": { + "q05_diff": 126.03125, + "q25_diff": 38.511719999998604, + "q50_diff": 31.43554999999833, + "q75_diff": 110.75586000000112, + "q95_diff": 415.25780999999915 + }, + "P4_ST_LD": { + "q05_diff": 0.610080000000039, + "q25_diff": 0.4074099999999703, + "q50_diff": 0.4549000000000092, + "q75_diff": 1.9834000000000174, + "q95_diff": 7.960319999999967 + }, + "P4_ST_PO": { + "q05_diff": 2.232030000000009, + "q25_diff": 0.7466999999999757, + "q50_diff": 0.598479999999995, + "q75_diff": 2.1484400000000505, + "q95_diff": 7.673490000000015 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 4.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 30.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 1.0, + "q50_diff": 3.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0189476811306442e-05, + "P1_B2016": 0.06984180370953874, + "P1_B3004": 1.018740504876714e-05, + "P1_B3005": 0.9999898125942444, + "P1_B4002": 0.9999898146591784, + "P1_B4005": 0.21093825435371383, + "P1_B400B": 0.7244630741315403, + "P1_B4022": 0.03037469907643997, + "P1_FCV02Z": 0.7097760903467218, + "P1_FCV03D": 0.058361009898620586, + "P1_FCV03Z": 0.05910243113731806, + "P1_FT01": 0.33235045462589885, + "P1_FT01Z": 0.47934193975762995, + "P1_FT02": 0.9626073770384019, + "P1_FT02Z": 0.9626623556483135, + "P1_FT03": 0.06070246033829663, + "P1_FT03Z": 0.06056540329368976, + "P1_LCV01D": 0.07855070257228525, + "P1_LCV01Z": 0.09172877445408123, + "P1_LIT01": 0.076707717410051, + "P1_PCV01D": 0.15038853814539088, + "P1_PCV01Z": 0.07032104675572548, + "P1_PCV02Z": 0.5669400491575541, + "P1_PIT01": 0.18925980718602764, + "P1_PIT02": 0.24850052045649773, + "P1_TIT01": 0.04032284159724764, + "P1_TIT02": 0.21903496669016953, + "P2_24Vdc": 0.10406769116973721, + "P2_CO_rpm": 0.3989935556927155, + "P2_HILout": 0.19361036990960512, + "P2_MSD": 0.0, + "P2_SIT01": 0.5984037633306999, + "P2_SIT02": 0.6277263410616808, + "P2_VT01": 0.029139879694105364, + "P2_VXT02": 0.026013214449299826, + "P2_VXT03": 0.01486567219627033, + "P2_VYT02": 0.02003540343450194, + "P2_VYT03": 0.008003689326990138, + "P3_FIT01": 0.4156848657804322, + "P3_LCP01D": 0.055904266923614165, + "P3_LCV01D": 0.05913608253912894, + "P3_LIT01": 0.031277935676521484, + "P3_PIT01": 0.27136737480366446, + "P4_HT_FD": 0.12580587902256582, + "P4_HT_LD": 0.15670838558697175, + "P4_HT_PO": 0.03321318464232814, + "P4_LD": 0.031089915069682283, + "P4_ST_FD": 0.23888891140340313, + "P4_ST_GOV": 0.08288957147549347, + "P4_ST_LD": 0.07543017974756916, + "P4_ST_PO": 0.09955759753764082, + "P4_ST_PT01": 0.31215475545109395, + "P4_ST_TT01": 0.12176489238930044 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09143925222566612, + "P1_FCV01Z": 0.7150479826839513, + "P1_FCV02D": 0.04372202231808956, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + }, + "avg_ks": 0.4049190251572327, + "avg_jsd": 0.0327003560472195, + "avg_lag1_diff": 0.23744482453266894 +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_file_condition__seed1337/eval_post.json b/example/results/ablations/runs/no_file_condition__seed1337/eval_post.json new file mode 100644 index 0000000..18ab78b --- /dev/null +++ b/example/results/ablations/runs/no_file_condition__seed1337/eval_post.json @@ -0,0 +1,971 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09355486979166669, + "std": 0.01799194293084728 + }, + "P1_B2016": { + "mean": 1.391534401041668, + "std": 0.13946106107922276 + }, + "P1_B3004": { + "mean": 403.2905014583335, + "std": 14.217563527951576 + }, + "P1_B3005": { + "mean": 1059.5253898958326, + "std": 58.42587703751758 + }, + "P1_B4002": { + "mean": 32.37490617187499, + "std": 0.6519533213854061 + }, + "P1_B4005": { + "mean": 64.99571291666655, + "std": 42.03467331899641 + }, + "P1_B400B": { + "mean": 1777.596925755208, + "std": 1195.6797141995014 + }, + "P1_B4022": { + "mean": 36.08671476562498, + "std": 0.7147782383668128 + }, + "P1_FCV02Z": { + "mean": 21.954832630208315, + "std": 38.87335152465215 + }, + "P1_FCV03D": { + "mean": 58.14541601562505, + "std": 6.267267768996455 + }, + "P1_FCV03Z": { + "mean": 58.21398343750003, + "std": 6.423211358701469 + }, + "P1_FT01": { + "mean": 194.69574812500008, + "std": 32.18984962125104 + }, + "P1_FT01Z": { + "mean": 875.3945892447917, + "std": 99.56005373645125 + }, + "P1_FT02": { + "mean": 1132.49033296875, + "std": 890.6858387444685 + }, + "P1_FT02Z": { + "mean": 1843.7261292187509, + "std": 1190.4241853713818 + }, + "P1_FT03": { + "mean": 282.27597710937533, + "std": 33.67135723115452 + }, + "P1_FT03Z": { + "mean": 1058.2564329947922, + "std": 59.50957255113764 + }, + "P1_LCV01D": { + "mean": 11.699055963541667, + "std": 3.0155193220433567 + }, + "P1_LCV01Z": { + "mean": 11.627510442708333, + "std": 3.0251072332459787 + }, + "P1_LIT01": { + "mean": 404.0009294010416, + "std": 15.70528322380496 + }, + "P1_PCV01D": { + "mean": 65.64593611979173, + "std": 23.56629129628467 + }, + "P1_PCV01Z": { + "mean": 65.565628671875, + "std": 23.790956943468952 + }, + "P1_PCV02Z": { + "mean": 11.867279765625007, + "std": 0.10811550492611105 + }, + "P1_PIT01": { + "mean": 1.3749083333333338, + "std": 0.059795921368924275 + }, + "P1_PIT02": { + "mean": 0.47586984374999997, + "std": 0.510092890346671 + }, + "P1_TIT01": { + "mean": 35.65237570312499, + "std": 0.5899993262509078 + }, + "P1_TIT02": { + "mean": 36.48672705729164, + "std": 1.220891267055021 + }, + "P2_24Vdc": { + "mean": 28.02860809895833, + "std": 0.0032970984602375326 + }, + "P2_CO_rpm": { + "mean": 54102.28249101564, + "std": 23.951768712470482 + }, + "P2_HILout": { + "mean": 711.4986896354167, + "std": 7.856828438379631 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.2530208333329, + "std": 5.357406258330356 + }, + "P2_SIT02": { + "mean": 783.9234037239586, + "std": 5.839501711639723 + }, + "P2_VT01": { + "mean": 11.923781276041664, + "std": 0.06898546777898959 + }, + "P2_VXT02": { + "mean": -3.299332369791668, + "std": 0.47724095306989195 + }, + "P2_VXT03": { + "mean": -1.3056848958333336, + "std": 0.35729730024065115 + }, + "P2_VYT02": { + "mean": 3.9808586718749996, + "std": 0.5057069185651452 + }, + "P2_VYT03": { + "mean": 6.233392708333337, + "std": 0.38918658337931306 + }, + "P3_FIT01": { + "mean": 959.5200457291662, + "std": 1756.2267796064482 + }, + "P3_LCP01D": { + "mean": 4590.908203072916, + "std": 5143.586765092662 + }, + "P3_LCV01D": { + "mean": 6167.148197187503, + "std": 7091.614281472208 + }, + "P3_LIT01": { + "mean": 13352.648086588542, + "std": 4370.825816197887 + }, + "P3_PIT01": { + "mean": 250.3358968229168, + "std": 490.47959019410433 + }, + "P4_HT_FD": { + "mean": -0.002084687499999999, + "std": 0.010185916807414626 + }, + "P4_HT_LD": { + "mean": 34.231568880208286, + "std": 33.60361854009684 + }, + "P4_HT_PO": { + "mean": 35.25451283854166, + "std": 31.650898099038645 + }, + "P4_LD": { + "mean": 366.1226154687502, + "std": 61.03142784766532 + }, + "P4_ST_FD": { + "mean": -5.903645833333325e-05, + "std": 0.004046068698763828 + }, + "P4_ST_GOV": { + "mean": 17918.506001718742, + "std": 1896.6960206854487 + }, + "P4_ST_LD": { + "mean": 332.09312765625, + "std": 38.5901339955848 + }, + "P4_ST_PO": { + "mean": 332.3664021614585, + "std": 35.26147496196201 + }, + "P4_ST_PT01": { + "mean": 10039.861785911464, + "std": 15.884781682619622 + }, + "P4_ST_TT01": { + "mean": 27608.10104877603, + "std": 24.783333022508646 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 1.2782299308983225e-05, + "std_abs_err": 6.494151245123314e-06 + }, + "P1_B2016": { + "mean_abs_err": 0.00026807637690096087, + "std_abs_err": 0.0036874986026596834 + }, + "P1_B3004": { + "mean_abs_err": 0.09556236619266656, + "std_abs_err": 1.2005860600902345 + }, + "P1_B3005": { + "mean_abs_err": 1.5233966327971302, + "std_abs_err": 2.757439473482698 + }, + "P1_B4002": { + "mean_abs_err": 0.05797985199510691, + "std_abs_err": 0.07327982218632823 + }, + "P1_B4005": { + "mean_abs_err": 4.470605900468627, + "std_abs_err": 0.14662412871327746 + }, + "P1_B400B": { + "mean_abs_err": 0.8789444227600143, + "std_abs_err": 2.3274412804287294 + }, + "P1_B4022": { + "mean_abs_err": 0.02554011822245883, + "std_abs_err": 0.04332738353199439 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.996102270263961, + "std_abs_err": 1.5651339146060295 + }, + "P1_FCV03D": { + "mean_abs_err": 0.42244743522177686, + "std_abs_err": 0.3322279497122995 + }, + "P1_FCV03Z": { + "mean_abs_err": 1.0850557426898035, + "std_abs_err": 0.29159260563013323 + }, + "P1_FT01": { + "mean_abs_err": 0.0279062312801841, + "std_abs_err": 4.841867782606716 + }, + "P1_FT01Z": { + "mean_abs_err": 3.6550922593607993, + "std_abs_err": 23.794350773133004 + }, + "P1_FT02": { + "mean_abs_err": 0.3278202658761984, + "std_abs_err": 1.4896748930558488 + }, + "P1_FT02Z": { + "mean_abs_err": 65.24768261010217, + "std_abs_err": 2.9211703997677887 + }, + "P1_FT03": { + "mean_abs_err": 0.3629931668535846, + "std_abs_err": 0.19714411985901847 + }, + "P1_FT03Z": { + "mean_abs_err": 2.785605353729352, + "std_abs_err": 1.6159163124796905 + }, + "P1_LCV01D": { + "mean_abs_err": 0.45527669823346884, + "std_abs_err": 0.1009043690132354 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.33856358662958463, + "std_abs_err": 0.061637735900667234 + }, + "P1_LIT01": { + "mean_abs_err": 0.11394276196466535, + "std_abs_err": 0.07338517543098888 + }, + "P1_PCV01D": { + "mean_abs_err": 1.8196376158996515, + "std_abs_err": 0.20347540462090308 + }, + "P1_PCV01Z": { + "mean_abs_err": 2.842523125027583, + "std_abs_err": 0.07364286719469249 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.024526819020080026, + "std_abs_err": 0.011271039461463683 + }, + "P1_PIT01": { + "mean_abs_err": 0.002572887420058123, + "std_abs_err": 0.0016366380009578269 + }, + "P1_PIT02": { + "mean_abs_err": 0.011287246591516753, + "std_abs_err": 0.01138125229179121 + }, + "P1_TIT01": { + "mean_abs_err": 0.028832539308226046, + "std_abs_err": 0.006718699064958811 + }, + "P1_TIT02": { + "mean_abs_err": 0.03825162104489266, + "std_abs_err": 0.0008674589692421186 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00010160366933931186, + "std_abs_err": 4.536765416938244e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.4277068670344306, + "std_abs_err": 1.496036937682117 + }, + "P2_HILout": { + "mean_abs_err": 0.035880365320508645, + "std_abs_err": 0.5978974388765312 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.650896580110043, + "std_abs_err": 0.15849089788613124 + }, + "P2_SIT02": { + "mean_abs_err": 0.019665841412347618, + "std_abs_err": 0.6489219548267569 + }, + "P2_VT01": { + "mean_abs_err": 3.436996846950535e-05, + "std_abs_err": 0.0004717857069875264 + }, + "P2_VXT02": { + "mean_abs_err": 0.00084610259832818, + "std_abs_err": 0.006208026798671662 + }, + "P2_VXT03": { + "mean_abs_err": 0.0012083908313442748, + "std_abs_err": 0.007669921076997488 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006101781179372878, + "std_abs_err": 0.005796465572357301 + }, + "P2_VYT03": { + "mean_abs_err": 0.0010124044645474228, + "std_abs_err": 0.009383621408221465 + }, + "P3_FIT01": { + "mean_abs_err": 191.38013037701694, + "std_abs_err": 23.49791818382664 + }, + "P3_LCP01D": { + "mean_abs_err": 6.1810548323419425, + "std_abs_err": 13.586767376783428 + }, + "P3_LCV01D": { + "mean_abs_err": 966.728519793739, + "std_abs_err": 251.68638565638685 + }, + "P3_LIT01": { + "mean_abs_err": 2.1964061054441117, + "std_abs_err": 16.65117926995117 + }, + "P3_PIT01": { + "mean_abs_err": 399.02200620041094, + "std_abs_err": 660.442340513091 + }, + "P4_HT_FD": { + "mean_abs_err": 0.001981499554648259, + "std_abs_err": 0.008153898493010526 + }, + "P4_HT_LD": { + "mean_abs_err": 0.23665180052267232, + "std_abs_err": 0.17982503009853446 + }, + "P4_HT_PO": { + "mean_abs_err": 1.2707050437847514, + "std_abs_err": 0.21002688171716954 + }, + "P4_LD": { + "mean_abs_err": 0.0061383254441125246, + "std_abs_err": 0.4156214782223131 + }, + "P4_ST_FD": { + "mean_abs_err": 2.992322845018443e-05, + "std_abs_err": 0.002403892440773799 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.544740471217665, + "std_abs_err": 44.552804697675356 + }, + "P4_ST_LD": { + "mean_abs_err": 0.09023048643229004, + "std_abs_err": 0.7369647039672529 + }, + "P4_ST_PO": { + "mean_abs_err": 0.0974308377997204, + "std_abs_err": 0.8105314595537081 + }, + "P4_ST_PT01": { + "mean_abs_err": 10.71337090761881, + "std_abs_err": 1.8999883426995083 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.9316429035025067, + "std_abs_err": 0.8660112179403185 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.6595583333333334, + "P1_B2016": 0.04325000000000001, + "P1_B3004": 0.671875, + "P1_B3005": 0.5293, + "P1_B4002": 0.7923208333333334, + "P1_B4005": 0.3062625, + "P1_B400B": 0.45285833333333336, + "P1_B4022": 0.5756333333333334, + "P1_FCV02Z": 0.7746, + "P1_FCV03D": 0.45162500000000005, + "P1_FCV03Z": 0.40985833333333327, + "P1_FT01": 0.4746, + "P1_FT01Z": 0.4750083333333333, + "P1_FT02": 0.4531416666666666, + "P1_FT02Z": 0.4029958333333334, + "P1_FT03": 0.5241, + "P1_FT03Z": 0.46950833333333336, + "P1_LCV01D": 0.3859916666666667, + "P1_LCV01Z": 0.397625, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5304583333333333, + "P1_PCV01Z": 0.5184416666666667, + "P1_PCV02Z": 0.6588541666666666, + "P1_PIT01": 0.16370833333333334, + "P1_PIT02": 0.3022, + "P1_TIT01": 0.10502083333333334, + "P1_TIT02": 0.09962083333333338, + "P2_24Vdc": 0.13548333333333334, + "P2_CO_rpm": 0.09815833333333329, + "P2_HILout": 0.08242499999999997, + "P2_MSD": 0.0, + "P2_SIT01": 0.48227083333333326, + "P2_SIT02": 0.5013166666666666, + "P2_VT01": 0.12296249999999997, + "P2_VXT02": 0.2815791666666666, + "P2_VXT03": 0.43657500000000005, + "P2_VYT02": 0.25022083333333334, + "P2_VYT03": 0.26955833333333334, + "P3_FIT01": 0.19270833333333337, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.5260416666666666, + "P3_LIT01": 0.09349166666666664, + "P3_PIT01": 0.339975, + "P4_HT_FD": 0.1423458333333334, + "P4_HT_LD": 0.06742500000000001, + "P4_HT_PO": 0.05970416666666667, + "P4_LD": 0.03121666666666667, + "P4_ST_FD": 0.023041666666666683, + "P4_ST_GOV": 0.03482083333333341, + "P4_ST_LD": 0.03254166666666658, + "P4_ST_PO": 0.035720833333333424, + "P4_ST_PT01": 0.24407916666666668, + "P4_ST_TT01": 0.07099583333333337 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.02845, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0011099999999999999, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.011359999999999815, + "q25_diff": 0.010220000000000118, + "q50_diff": 0.005260000000000042, + "q75_diff": 0.014270000000000005, + "q95_diff": 0.030689999999999884 + }, + "P1_B3004": { + "q05_diff": 14.80313000000001, + "q25_diff": 1.6197799999999916, + "q50_diff": 6.726189999999974, + "q75_diff": 8.508819999999957, + "q95_diff": 37.86444 + }, + "P1_B3005": { + "q05_diff": 40.40789999999993, + "q25_diff": 8.045049999999947, + "q50_diff": 89.51202, + "q75_diff": 104.97393, + "q95_diff": 4.4593499999998585 + }, + "P1_B4002": { + "q05_diff": 0.4564699999999995, + "q25_diff": 1.6555000000000035, + "q50_diff": 1.6555000000000035, + "q75_diff": 0.9287000000000063, + "q95_diff": 0.16969000000000278 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 81.96193, + "q50_diff": 0.04000000000000625, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2478.8725, + "q50_diff": 192.26342999999997, + "q75_diff": 9.545890000000327, + "q95_diff": 7.019040000000132 + }, + "P1_B4022": { + "q05_diff": 0.38222999999999985, + "q25_diff": 1.3479000000000028, + "q50_diff": 1.4273799999999994, + "q75_diff": 0.9127399999999994, + "q95_diff": 0.5270299999999963 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.05340999999999996, + "q75_diff": 38.169869999999996, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 1.3654200000000003, + "q25_diff": 0.2554099999999977, + "q50_diff": 8.60333, + "q75_diff": 10.84863, + "q95_diff": 0.824300000000008 + }, + "P1_FCV03Z": { + "q05_diff": 1.7547599999999974, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.148199999999996, + "q75_diff": 10.864249999999998, + "q95_diff": 0.9307899999999876 + }, + "P1_FT01": { + "q05_diff": 13.542220000000015, + "q25_diff": 8.20162000000002, + "q50_diff": 32.806329999999974, + "q75_diff": 37.95618999999999, + "q95_diff": 2.861050000000006 + }, + "P1_FT01Z": { + "q05_diff": 38.05517999999995, + "q25_diff": 19.59997999999996, + "q50_diff": 74.79840000000002, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.14705, + "q50_diff": 257.5476000000001, + "q75_diff": 12.970209999999952, + "q95_diff": 10.108890000000201 + }, + "P1_FT02Z": { + "q05_diff": 1.9862300000000026, + "q25_diff": 2300.54303, + "q50_diff": 14.318849999999657, + "q75_diff": 7.299800000000232, + "q95_diff": 6.317140000000109 + }, + "P1_FT03": { + "q05_diff": 36.18037000000001, + "q25_diff": 1.363020000000006, + "q50_diff": 51.52633000000003, + "q75_diff": 58.58354000000003, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 23.41980000000001, + "q25_diff": 2.9805299999999306, + "q50_diff": 84.98266000000012, + "q75_diff": 100.6383699999999, + "q95_diff": 3.5079399999999623 + }, + "P1_LCV01D": { + "q05_diff": 0.17497000000000007, + "q25_diff": 0.29953000000000074, + "q50_diff": 4.171619999999999, + "q75_diff": 3.162939999999999, + "q95_diff": 0.7125800000000009 + }, + "P1_LCV01Z": { + "q05_diff": 0.0, + "q25_diff": 0.5874600000000001, + "q50_diff": 4.638670000000001, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.9536700000000007 + }, + "P1_LIT01": { + "q05_diff": 14.325409999999977, + "q25_diff": 1.04367000000002, + "q50_diff": 6.2459499999999935, + "q75_diff": 8.229829999999993, + "q95_diff": 34.29950000000002 + }, + "P1_PCV01D": { + "q05_diff": 0.18625000000000114, + "q25_diff": 2.7490099999999984, + "q50_diff": 18.274499999999996, + "q75_diff": 43.266600000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.22888000000000375, + "q25_diff": 2.0370500000000007, + "q50_diff": 16.029360000000004, + "q75_diff": 43.33495, + "q95_diff": 45.1493 + }, + "P1_PCV02Z": { + "q05_diff": 0.2288800000000002, + "q25_diff": 0.2212499999999995, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_PIT01": { + "q05_diff": 0.03493000000000013, + "q25_diff": 0.017550000000000177, + "q50_diff": 0.0045800000000000285, + "q75_diff": 0.024410000000000043, + "q95_diff": 0.030510000000000037 + }, + "P1_PIT02": { + "q05_diff": 0.03128, + "q25_diff": 0.06789999999999999, + "q50_diff": 0.0015299999999999758, + "q75_diff": 0.08316000000000001, + "q95_diff": 1.52573 + }, + "P1_TIT01": { + "q05_diff": 0.015270000000001005, + "q25_diff": 0.0457800000000006, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.27465999999999724, + "q95_diff": 0.045769999999997424 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.06367000000000189, + "q50_diff": 0.21363000000000198, + "q75_diff": 0.3044799999999981, + "q95_diff": 0.6713900000000024 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.00013999999999825263, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.00010999999999938836 + }, + "P2_CO_rpm": { + "q05_diff": 10.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 3.0, + "q95_diff": 0.0 + }, + "P2_HILout": { + "q05_diff": 0.3295900000000529, + "q25_diff": 1.6540499999999838, + "q50_diff": 0.31737999999995736, + "q75_diff": 1.7334000000000742, + "q95_diff": 2.2399900000000343 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 1.0, + "q25_diff": 3.0, + "q50_diff": 5.0, + "q75_diff": 6.0, + "q95_diff": 5.0 + }, + "P2_SIT02": { + "q05_diff": 2.657410000000027, + "q25_diff": 3.569340000000011, + "q50_diff": 5.614020000000096, + "q75_diff": 6.363950000000045, + "q95_diff": 6.839899999999943 + }, + "P2_VT01": { + "q05_diff": 0.008340000000000458, + "q25_diff": 0.015240000000000364, + "q50_diff": 0.01422999999999952, + "q75_diff": 0.013299999999999201, + "q95_diff": 0.019059999999999633 + }, + "P2_VXT02": { + "q05_diff": 0.13016000000000005, + "q25_diff": 0.29947999999999997, + "q50_diff": 0.28572, + "q75_diff": 0.2670600000000003, + "q95_diff": 0.45709 + }, + "P2_VXT03": { + "q05_diff": 0.09953999999999996, + "q25_diff": 0.26919000000000004, + "q50_diff": 0.29715999999999987, + "q75_diff": 0.32733999999999996, + "q95_diff": 0.6096 + }, + "P2_VYT02": { + "q05_diff": 0.09771999999999981, + "q25_diff": 0.26203999999999983, + "q50_diff": 0.22734000000000032, + "q75_diff": 0.2356600000000002, + "q95_diff": 0.3893300000000002 + }, + "P2_VYT03": { + "q05_diff": 0.05701999999999963, + "q25_diff": 0.1859200000000003, + "q50_diff": 0.17567999999999984, + "q75_diff": 0.18238999999999983, + "q95_diff": 0.3547399999999996 + }, + "P3_FIT01": { + "q05_diff": 5.0, + "q25_diff": 2.0, + "q50_diff": 67.0, + "q75_diff": 2512.0, + "q95_diff": 17.22606999999971 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 90.70459000000005, + "q75_diff": 123.05469000000085, + "q95_diff": 12.901369999999588 + }, + "P3_LCV01D": { + "q05_diff": 16.0, + "q25_diff": 336.0, + "q50_diff": 9488.0, + "q75_diff": 976.0, + "q95_diff": 31.57227000000057 + }, + "P3_LIT01": { + "q05_diff": 1132.3847699999997, + "q25_diff": 26.040039999999863, + "q50_diff": 642.8085900000005, + "q75_diff": 1150.1914099999995, + "q95_diff": 4.4375 + }, + "P3_PIT01": { + "q05_diff": 4.0, + "q25_diff": 0.0, + "q50_diff": 1.0, + "q75_diff": 929.0, + "q95_diff": 2364.0 + }, + "P4_HT_FD": { + "q05_diff": 6.999999999999997e-05, + "q25_diff": 0.00014, + "q50_diff": 7.000000000000002e-05, + "q75_diff": 0.0, + "q95_diff": 0.00029 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 27.14122, + "q75_diff": 0.13020999999999106, + "q95_diff": 0.21701000000000192 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 0.9223299999999999, + "q50_diff": 8.89752, + "q75_diff": 0.054199999999994475, + "q95_diff": 0.6510700000000043 + }, + "P4_LD": { + "q05_diff": 0.5106200000000172, + "q25_diff": 3.088860000000011, + "q50_diff": 3.4461600000000203, + "q75_diff": 0.07796999999999343, + "q95_diff": 4.251340000000027 + }, + "P4_ST_FD": { + "q05_diff": 1.0000000000000026e-05, + "q25_diff": 0.0, + "q50_diff": 7.999999999999999e-05, + "q75_diff": 0.0, + "q95_diff": 8.000000000000021e-05 + }, + "P4_ST_GOV": { + "q05_diff": 126.03125, + "q25_diff": 38.511719999998604, + "q50_diff": 31.43554999999833, + "q75_diff": 110.75586000000112, + "q95_diff": 415.25780999999915 + }, + "P4_ST_LD": { + "q05_diff": 0.610080000000039, + "q25_diff": 0.4074099999999703, + "q50_diff": 0.4549000000000092, + "q75_diff": 1.9834000000000174, + "q95_diff": 7.960319999999967 + }, + "P4_ST_PO": { + "q05_diff": 2.232030000000009, + "q25_diff": 0.7466999999999757, + "q50_diff": 0.598479999999995, + "q75_diff": 2.1484400000000505, + "q95_diff": 7.673490000000015 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 4.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 30.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 1.0, + "q50_diff": 3.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0624831607922858, + "P1_B2016": 0.06984180370953874, + "P1_B3004": 0.9420514424699565, + "P1_B3005": 0.9755864990985265, + "P1_B4002": 0.9939428775878422, + "P1_B4005": 1.0901635813805555, + "P1_B400B": 0.7244630741315403, + "P1_B4022": 1.0728287547128275, + "P1_FCV02Z": 1.0856923346443097, + "P1_FCV03D": 1.0287616800381814, + "P1_FCV03Z": 1.01920928123416, + "P1_FT01": 0.33235045462589885, + "P1_FT01Z": 0.47934193975762995, + "P1_FT02": 0.9626073770384019, + "P1_FT02Z": 1.0572315062387077, + "P1_FT03": 0.06070246033829663, + "P1_FT03Z": 0.9861265787637634, + "P1_LCV01D": 0.9922897795117445, + "P1_LCV01Z": 0.9035160861736835, + "P1_LIT01": 0.076707717410051, + "P1_PCV01D": 0.9847169107083952, + "P1_PCV01Z": 1.1483081333334428, + "P1_PCV02Z": 0.5764046458797007, + "P1_PIT01": 0.18925980718602764, + "P1_PIT02": 0.24850052045649773, + "P1_TIT01": 0.04032284159724764, + "P1_TIT02": 0.21903496669016953, + "P2_24Vdc": 0.039965792976989475, + "P2_CO_rpm": 0.3989935556927155, + "P2_HILout": 0.24706059027478758, + "P2_MSD": 0.0, + "P2_SIT01": 0.5984037633306999, + "P2_SIT02": 0.6277263410616808, + "P2_VT01": 0.029139879694105364, + "P2_VXT02": 0.026013214449299826, + "P2_VXT03": 0.01486567219627033, + "P2_VYT02": 0.02003540343450194, + "P2_VYT03": 0.008003689326990138, + "P3_FIT01": 0.4156848657804322, + "P3_LCP01D": 0.055904266923614165, + "P3_LCV01D": 0.05913608253912894, + "P3_LIT01": 0.031277935676521484, + "P3_PIT01": 0.27136737480366446, + "P4_HT_FD": 0.12580587902256582, + "P4_HT_LD": 1.052071148480924, + "P4_HT_PO": 1.0918064404320895, + "P4_LD": 0.031089915069682283, + "P4_ST_FD": 0.23888891140340313, + "P4_ST_GOV": 0.08288957147549347, + "P4_ST_LD": 0.07543017974756916, + "P4_ST_PO": 0.09955759753764082, + "P4_ST_PT01": 0.31215475545109395, + "P4_ST_TT01": 0.12176489238930044 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09143925222566612, + "P1_FCV01Z": 0.7150479826839513, + "P1_FCV02D": 0.04372202231808956, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_post_calibration__seed1337/comprehensive_eval.json b/example/results/ablations/runs/no_post_calibration__seed1337/comprehensive_eval.json new file mode 100644 index 0000000..3ad5632 --- /dev/null +++ b/example/results/ablations/runs/no_post_calibration__seed1337/comprehensive_eval.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_post_calibration__seed1337/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.5430239779874214, + "avg_jsd": 0.024122590895914533, + "avg_lag1_diff": 0.25321129348110133 + }, + "two_sample": { + "continuous_mmd_rbf": 0.7487249374389648, + "continuous_mmd_gamma": 1.88790421246002e-05, + "histogram_mmd_rbf": 0.7487199306488037, + "histogram_mmd_gamma": 1.8877988243585336e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 143.29190063476562, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 2.5969793603566886, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.24867647886276245, + "corr_frobenius": 19.313549041748047, + "lag1_corr_mean_abs_diff": 0.24658013880252838, + "lag1_corr_frobenius": 19.289058685302734, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.3277365267276764, + "corr_frobenius": 11.602654457092285, + "lag1_corr_mean_abs_diff": 0.3405558466911316, + "lag1_corr_frobenius": 11.9528169631958 + }, + "P2": { + "corr_mean_abs_diff": 0.12732942402362823, + "corr_frobenius": 2.7223596572875977, + "lag1_corr_mean_abs_diff": 0.15835681557655334, + "lag1_corr_frobenius": 2.862332582473755 + }, + "P3": { + "corr_mean_abs_diff": 0.47702693939208984, + "corr_frobenius": 3.1172080039978027, + "lag1_corr_mean_abs_diff": 0.5540300607681274, + "lag1_corr_frobenius": 3.3840389251708984 + }, + "P4": { + "corr_mean_abs_diff": 0.15865422785282135, + "corr_frobenius": 2.264617681503296, + "lag1_corr_mean_abs_diff": 0.16694416105747223, + "lag1_corr_frobenius": 2.34615159034729 + } + } + }, + "frequency": { + "avg_psd_l1": 0.019222214818000793, + "avg_psd_cosine": 0.34924665231296786, + "avg_low_high_ratio_abs_diff": 7547180.663933763 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 15.0, + "mean_dwell": 24.0, + "median_dwell": 2.5, + "mean_step": 68.42660395304362, + "median_step": 1.9043693542480469 + }, + "P1_B2004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B3004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.01115664489586733, + "median_step": 0.004810333251953125 + }, + "P1_B3005": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22998.14285714286, + "mean_abs_diff_mean_dwell": 14201.625685203559, + "mean_abs_diff_median_dwell": 14207.642857142857, + "mean_abs_diff_mean_step": 33.693117622162895, + "mean_abs_diff_median_step": 0.831573486328125 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.9505208333333334, + "change_rate": 0.06266318537859007, + "step_median": 5.5986480712890625 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.36156209309895837, + "mean_abs_diff_change_rate": 0.3981569320135546, + "mean_abs_diff_step_median": 5.566627770662308 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.0026041666666666665, + "top1_mass": 1.0, + "top3_mass": 1.0, + "median_dwell": 384.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.5364583333333334, + "top1_mass": 0.044270833333333336, + "top3_mass": 0.11197916666666667, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.515625, + "top1_mass": 0.026041666666666668, + "top3_mass": 0.06770833333333333, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.026041666666666668, + "top1_mass": 0.3776041666666667, + "top3_mass": 0.6901041666666666, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.3046875, + "top1_mass": 0.0546875, + "top3_mass": 0.15364583333333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.2994791666666667, + "top1_mass": 0.1328125, + "top3_mass": 0.23958333333333334, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.3385416666666667, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.09114583333333333, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.21354166666666666, + "top1_mass": 0.2708333333333333, + "top3_mass": 0.4375, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.25582249959309894, + "mean_abs_diff_top1_mass": 0.12969080607096353, + "mean_abs_diff_top3_mass": 0.12766138712565103, + "mean_abs_diff_median_dwell": 47.75 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.27557000517845154, + "q50": 0.27557000517845154, + "q95": 0.2763400077819824, + "tail_ratio": NaN + }, + "P2_SIT02": { + "q05": 780.9140625, + "q50": 786.1126708984375, + "q95": 789.8550415039062, + "tail_ratio": 0.7198793058914692 + }, + "P1_FT03": { + "q05": 256.55377197265625, + "q50": 290.628173828125, + "q95": 296.7215270996094, + "tail_ratio": 0.17882495186064215 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 21.466984341541927, + "mean_abs_diff_q50": 5.517051508029302, + "mean_abs_diff_q95": 7.798035264015198, + "mean_abs_diff_tail_ratio": 0.05002319332860432 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 381.0, + "mean_dwell": 1.0052356020942408, + "median_dwell": 1.0, + "mean_step": 6.775410729756192, + "median_step": 2.187347412109375 + }, + "P1_FT02Z": { + "num_changes": 382.0, + "mean_dwell": 1.0026109660574412, + "median_dwell": 1.0, + "mean_step": 179.98989802255682, + "median_step": 6.797607421875 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89281.0, + "mean_abs_diff_mean_dwell": 0.09700721625641795, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 85.7315538006411, + "mean_abs_diff_median_step": 1.2255706787109375 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 4.248061656951904, + "std": 6.320553779602051, + "lag1": 0.5254456996917725 + }, + "P2_24Vdc": { + "mean": 28.027786254882812, + "std": 0.003147909650579095, + "lag1": 0.18862749636173248 + }, + "P2_HILout": { + "mean": 713.849609375, + "std": 11.860054016113281, + "lag1": 0.05019346624612808 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 10.662841320037842, + "mean_abs_diff_std": 9.540577205674103, + "mean_abs_diff_lag1": 0.27129685257871944 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 1.0864471197128296, + "mae": 0.8489447236061096 + }, + "real_plus_synthetic": { + "rmse": 0.5498619675636292, + "mae": 0.36013707518577576 + }, + "rmse_ratio_synth_to_real": 1.945814145690229 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8084183673469387, + "auprc": 0.5763650635885282, + "threshold": 0.010592951439321041, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6666666666666667, + "best_f1_threshold": 1.5030077695846558 + }, + "real_plus_synthetic": { + "auroc": 0.8835575139146568, + "auprc": 0.6469108348000047, + "threshold": 0.34942540526390076, + "f1": 0.2966507177033493, + "precision": 0.18787878787878787, + "recall": 0.7045454545454546, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.5489799380302429 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_post_calibration__seed1337/comprehensive_eval_post.json b/example/results/ablations/runs/no_post_calibration__seed1337/comprehensive_eval_post.json new file mode 100644 index 0000000..9c757a0 --- /dev/null +++ b/example/results/ablations/runs/no_post_calibration__seed1337/comprehensive_eval_post.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_post_calibration__seed1337/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.255412220954895, + "continuous_mmd_gamma": 5.599573207529589e-05, + "histogram_mmd_rbf": 0.25547337532043457, + "histogram_mmd_gamma": 5.597532799956969e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 75.49688720703125, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 1.368282903494881, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.2054588943719864, + "corr_frobenius": 17.007688522338867, + "lag1_corr_mean_abs_diff": 0.21240532398223877, + "lag1_corr_frobenius": 17.63569450378418, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.350882351398468, + "corr_frobenius": 12.301260948181152, + "lag1_corr_mean_abs_diff": 0.3749401867389679, + "lag1_corr_frobenius": 13.010165214538574 + }, + "P2": { + "corr_mean_abs_diff": 0.09124420583248138, + "corr_frobenius": 1.8073540925979614, + "lag1_corr_mean_abs_diff": 0.15456748008728027, + "lag1_corr_frobenius": 2.887573719024658 + }, + "P3": { + "corr_mean_abs_diff": 0.47702693939208984, + "corr_frobenius": 3.1172080039978027, + "lag1_corr_mean_abs_diff": 0.5540300607681274, + "lag1_corr_frobenius": 3.3840389251708984 + }, + "P4": { + "corr_mean_abs_diff": 0.21809031069278717, + "corr_frobenius": 3.3016245365142822, + "lag1_corr_mean_abs_diff": 0.23964621126651764, + "lag1_corr_frobenius": 3.604497194290161 + } + } + }, + "frequency": { + "avg_psd_l1": 0.022475097328424454, + "avg_psd_cosine": 0.4674628899907166, + "avg_low_high_ratio_abs_diff": 16.7642201614336 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 308.0, + "mean_dwell": 1.2427184466019416, + "median_dwell": 1.0, + "mean_step": 0.8413416007896523, + "median_step": 0.7274894714355469 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 347.0, + "mean_dwell": 1.103448275862069, + "median_dwell": 1.0, + "mean_step": 41.3141472878254, + "median_step": 54.116028785705566 + }, + "P1_B2004": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 0.014068839402653235, + "median_step": 0.0017450004816055298 + }, + "P1_B3004": { + "num_changes": 370.0, + "mean_dwell": 1.0350404312668464, + "median_dwell": 1.0, + "mean_step": 14.77256890374261, + "median_step": 10.804779052734375 + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.8157603410765646, + "median_step": 0.5944099426269531 + }, + "P1_B3005": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 64.28072175926926, + "median_step": 41.228179931640625 + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22997.85714285714, + "mean_abs_diff_mean_dwell": 14417.281142645239, + "mean_abs_diff_median_dwell": 13988.57142857143, + "mean_abs_diff_mean_step": 19.03385039702909, + "mean_abs_diff_median_step": 16.840154950817425 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6822916666666666, + "change_rate": 0.7702349869451697, + "step_median": 68.81373023986816 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.09333292643229163, + "mean_abs_diff_change_rate": 0.309414869553025, + "mean_abs_diff_step_median": 68.78170993924141 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.015625, + "top1_mass": 0.40625, + "top3_mass": 0.7760416666666666, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8776041666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.059895833333333336, + "top3_mass": 0.0703125, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.18229166666666666, + "top3_mass": 0.3697916666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.8463541666666666, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4817708333333333, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.7864583333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3515625, + "top1_mass": 0.036458333333333336, + "top3_mass": 0.09635416666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5303484598795573, + "mean_abs_diff_top1_mass": 0.03112411499023438, + "mean_abs_diff_top3_mass": 0.05629857381184896, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.27557000517845154, + "q50": 0.27557000517845154, + "q95": 0.2763400077819824, + "tail_ratio": NaN + }, + "P2_SIT02": { + "q05": 780.9140625, + "q50": 786.1126708984375, + "q95": 789.8550415039062, + "tail_ratio": 0.7198793058914692 + }, + "P1_FT03": { + "q05": 256.55377197265625, + "q50": 290.628173828125, + "q95": 296.7215270996094, + "tail_ratio": 0.17882495186064215 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 21.466984341541927, + "mean_abs_diff_q50": 5.517051508029302, + "mean_abs_diff_q95": 7.798035264015198, + "mean_abs_diff_tail_ratio": 0.05002319332860432 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 64.74391126130756, + "median_step": 62.410614013671875 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 1312.3134692358592, + "median_step": 1284.6439208984375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.09039718638964156, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 680.877589673068, + "mean_abs_diff_median_step": 670.2603607177734 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 35.254512786865234, + "std": 31.650897979736328, + "lag1": -0.0927804708480835 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.003297113813459873, + "lag1": 0.06643600016832352 + }, + "P2_HILout": { + "mean": 711.4987182617188, + "std": 7.856828212738037, + "lag1": -0.013758387416601181 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.4565467834472656, + "mean_abs_diff_std": 0.2373291968833655, + "mean_abs_diff_lag1": 0.4579590285817782 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9957595467567444, + "mae": 0.7932217717170715 + }, + "real_plus_synthetic": { + "rmse": 0.5611229538917542, + "mae": 0.36314624547958374 + }, + "rmse_ratio_synth_to_real": 1.783393758085071 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8202458256029683, + "auprc": 0.5775913158725147, + "threshold": 0.001364508643746376, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6567164179104478, + "best_f1_threshold": 2.473949432373047 + }, + "real_plus_synthetic": { + "auroc": 0.8820500927643784, + "auprc": 0.6458542095404818, + "threshold": 0.30257150530815125, + "f1": 0.2471042471042471, + "precision": 0.14883720930232558, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.5442057847976685 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_post_calibration__seed1337/eval.json b/example/results/ablations/runs/no_post_calibration__seed1337/eval.json new file mode 100644 index 0000000..9cf337d --- /dev/null +++ b/example/results/ablations/runs/no_post_calibration__seed1337/eval.json @@ -0,0 +1,974 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.11384, + "std": 0.0 + }, + "P1_B2016": { + "mean": 1.226465963541668, + "std": 0.05762137409951418 + }, + "P1_B3004": { + "mean": 396.00211, + "std": 0.0 + }, + "P1_B3005": { + "mean": 1015.43976, + "std": 0.0 + }, + "P1_B4002": { + "mean": 33.54375, + "std": 0.0 + }, + "P1_B4005": { + "mean": 99.37991791666676, + "std": 3.3420495587851917 + }, + "P1_B400B": { + "mean": 2770.5973602343743, + "std": 241.08970370015072 + }, + "P1_B4022": { + "mean": 36.881583541666664, + "std": 0.14105102171131376 + }, + "P1_FCV02Z": { + "mean": -1.8481197916666667, + "std": 0.02030578567973585 + }, + "P1_FCV03D": { + "mean": 52.16431437500002, + "std": 2.175644230182644 + }, + "P1_FCV03Z": { + "mean": 58.81612481770837, + "std": 1.410851739333414 + }, + "P1_FT01": { + "mean": 173.78418085937486, + "std": 10.871636774164493 + }, + "P1_FT01Z": { + "mean": 832.3371609895837, + "std": 21.14366010462666 + }, + "P1_FT02": { + "mean": 1856.0462919531253, + "std": 368.80879559858437 + }, + "P1_FT02Z": { + "mean": 1856.0462919531253, + "std": 368.80879559858437 + }, + "P1_FT03": { + "mean": 280.8603020833332, + "std": 14.266500816305975 + }, + "P1_FT03Z": { + "mean": 280.8603020833332, + "std": 14.266500816305975 + }, + "P1_LCV01D": { + "mean": 8.505994375000006, + "std": 1.3665904356079666 + }, + "P1_LCV01Z": { + "mean": 8.192170807291667, + "std": 1.4691896837147378 + }, + "P1_LIT01": { + "mean": 398.16104973958295, + "std": 2.558201454443084 + }, + "P1_PCV01D": { + "mean": 40.075800468749996, + "std": 5.807058131429667 + }, + "P1_PCV01Z": { + "mean": 42.31939664062498, + "std": 5.749748963937845 + }, + "P1_PCV02Z": { + "mean": 11.78894, + "std": 0.0 + }, + "P1_PIT01": { + "mean": 1.3642693489583342, + "std": 0.0045464753907211814 + }, + "P1_PIT02": { + "mean": 0.27590526041666674, + "std": 0.0003986936041241994 + }, + "P1_TIT01": { + "mean": 35.183942265625014, + "std": 0.25176049384259186 + }, + "P1_TIT02": { + "mean": 35.410734296875, + "std": 0.34275359801312044 + }, + "P2_24Vdc": { + "mean": 28.027787161458317, + "std": 0.0031478451355437054 + }, + "P2_CO_rpm": { + "mean": 54109.52951054688, + "std": 21.97475672644607 + }, + "P2_HILout": { + "mean": 713.8496116927085, + "std": 11.860054158208046 + }, + "P2_MSD": { + "mean": 758.4425, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 784.7902604166661, + "std": 3.249294028147303 + }, + "P2_SIT02": { + "mean": 785.8944788281248, + "std": 2.817230183482184 + }, + "P2_VT01": { + "mean": 11.922679921875003, + "std": 0.05504070160484108 + }, + "P2_VXT02": { + "mean": -3.3796592708333324, + "std": 0.3392395874187618 + }, + "P2_VXT03": { + "mean": -1.396118854166667, + "std": 0.18398561380285178 + }, + "P2_VYT02": { + "mean": 4.004463072916667, + "std": 0.389492910854868 + }, + "P2_VYT03": { + "mean": 6.30108776041666, + "std": 0.24693428433774522 + }, + "P3_FIT01": { + "mean": -19.732960494791662, + "std": 16.706026566947937 + }, + "P3_LCP01D": { + "mean": 8243.110082057286, + "std": 3711.12057106618 + }, + "P3_LCV01D": { + "mean": 1076.3946400000002, + "std": 1690.1832719119786 + }, + "P3_LIT01": { + "mean": 11849.491632994792, + "std": 1037.1359212752477 + }, + "P3_PIT01": { + "mean": -17.142703098958336, + "std": 3.7681811558923877 + }, + "P4_HT_FD": { + "mean": 0.000783671875, + "std": 0.0007698905056872621 + }, + "P4_HT_LD": { + "mean": -115.69799739583333, + "std": 35.27327640165136 + }, + "P4_HT_PO": { + "mean": 4.2480617708333295, + "std": 6.320553823365855 + }, + "P4_LD": { + "mean": 317.36853927083354, + "std": 15.991101621760016 + }, + "P4_ST_FD": { + "mean": -0.0005309895833333335, + "std": 0.0014984758348293276 + }, + "P4_ST_GOV": { + "mean": 15550.608222916668, + "std": 754.5835177329105 + }, + "P4_ST_LD": { + "mean": 281.8861859374999, + "std": 13.154505490055021 + }, + "P4_ST_PO": { + "mean": 289.0518683072917, + "std": 14.117132016051908 + }, + "P4_ST_PT01": { + "mean": 10046.702702838544, + "std": 6.4608246856850355 + }, + "P4_ST_TT01": { + "mean": 27597.026677473943, + "std": 19.624487126975794 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.020272347909024324, + "std_abs_err": 0.017998437082092405 + }, + "P1_B2016": { + "mean_abs_err": 0.16480036112309904, + "std_abs_err": 0.0781521883770489 + }, + "P1_B3004": { + "mean_abs_err": 7.383953824526145, + "std_abs_err": 15.41814958804181 + }, + "P1_B3005": { + "mean_abs_err": 45.609026528629784, + "std_abs_err": 61.18331651100028 + }, + "P1_B4002": { + "mean_abs_err": 1.2268236801201198, + "std_abs_err": 0.5786734991990778 + }, + "P1_B4005": { + "mean_abs_err": 38.854810900468834, + "std_abs_err": 38.8392478889245 + }, + "P1_B400B": { + "mean_abs_err": 992.1214900564062, + "std_abs_err": 952.262569218922 + }, + "P1_B4022": { + "mean_abs_err": 0.8204088942641405, + "std_abs_err": 0.5303998331235046 + }, + "P1_FCV02Z": { + "mean_abs_err": 26.79905469213894, + "std_abs_err": 40.41817965357844 + }, + "P1_FCV03D": { + "mean_abs_err": 6.403549075846811, + "std_abs_err": 4.423851488526111 + }, + "P1_FCV03Z": { + "mean_abs_err": 0.48291436248146624, + "std_abs_err": 5.303952224998188 + }, + "P1_FT01": { + "mean_abs_err": 20.939473496905407, + "std_abs_err": 16.476345064479833 + }, + "P1_FT01Z": { + "mean_abs_err": 46.71252051456884, + "std_abs_err": 54.622042858691586 + }, + "P1_FT02": { + "mean_abs_err": 723.2281387184992, + "std_abs_err": 520.3873682528283 + }, + "P1_FT02Z": { + "mean_abs_err": 77.56784534447661, + "std_abs_err": 824.5365601725653 + }, + "P1_FT03": { + "mean_abs_err": 1.7786681928957364, + "std_abs_err": 19.20771229498953 + }, + "P1_FT03Z": { + "mean_abs_err": 780.1817362651884, + "std_abs_err": 46.85898804731136 + }, + "P1_LCV01D": { + "mean_abs_err": 3.6483382867751306, + "std_abs_err": 1.5480245174221547 + }, + "P1_LCV01Z": { + "mean_abs_err": 3.7739032220462505, + "std_abs_err": 1.4942798136305737 + }, + "P1_LIT01": { + "mean_abs_err": 5.953822423423333, + "std_abs_err": 13.073696593930887 + }, + "P1_PCV01D": { + "mean_abs_err": 27.389773266941383, + "std_abs_err": 17.962708569475907 + }, + "P1_PCV01Z": { + "mean_abs_err": 26.088755156277607, + "std_abs_err": 18.1148508467258 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.05381294660492664, + "std_abs_err": 0.09684446546464737 + }, + "P1_PIT01": { + "mean_abs_err": 0.013211871795057784, + "std_abs_err": 0.05688608397916092 + }, + "P1_PIT02": { + "mean_abs_err": 0.21125182992484998, + "std_abs_err": 0.49831294445075563 + }, + "P1_TIT01": { + "mean_abs_err": 0.49726597680820106, + "std_abs_err": 0.3315201333433571 + }, + "P1_TIT02": { + "mean_abs_err": 1.114244381461532, + "std_abs_err": 0.8790051280111426 + }, + "P2_24Vdc": { + "mean_abs_err": 0.0009225411693520869, + "std_abs_err": 0.00019462097886320962 + }, + "P2_CO_rpm": { + "mean_abs_err": 6.819312664207246, + "std_abs_err": 0.48097504834229454 + }, + "P2_HILout": { + "mean_abs_err": 2.3868024226123907, + "std_abs_err": 3.405328280951883 + }, + "P2_MSD": { + "mean_abs_err": 4.750739999999951, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.8863430032231463, + "std_abs_err": 1.949621332296922 + }, + "P2_SIT02": { + "mean_abs_err": 1.9907409455785228, + "std_abs_err": 2.3733495733307826 + }, + "P2_VT01": { + "mean_abs_err": 0.0011357241351301894, + "std_abs_err": 0.013472980467160986 + }, + "P2_VXT02": { + "mean_abs_err": 0.07948079844333611, + "std_abs_err": 0.13179333885245847 + }, + "P2_VXT03": { + "mean_abs_err": 0.08922556750198907, + "std_abs_err": 0.16564176536080188 + }, + "P2_VYT02": { + "mean_abs_err": 0.02299422292372988, + "std_abs_err": 0.11041754213791988 + }, + "P2_VYT03": { + "mean_abs_err": 0.06668264761877563, + "std_abs_err": 0.13286867763334637 + }, + "P3_FIT01": { + "mean_abs_err": 1170.633136600975, + "std_abs_err": 1763.018671223327 + }, + "P3_LCP01D": { + "mean_abs_err": 3658.3829338167125, + "std_abs_err": 1418.8794266496984 + }, + "P3_LCV01D": { + "mean_abs_err": 6057.482076981241, + "std_abs_err": 5149.744623903843 + }, + "P3_LIT01": { + "mean_abs_err": 1505.3528596991946, + "std_abs_err": 3317.0387156526886 + }, + "P3_PIT01": { + "mean_abs_err": 666.5006061222861, + "std_abs_err": 1147.153749551303 + }, + "P4_HT_FD": { + "mean_abs_err": 0.0008868598203517398, + "std_abs_err": 0.0012621278087168383 + }, + "P4_HT_LD": { + "mean_abs_err": 149.69291447551893, + "std_abs_err": 1.8494828916530537 + }, + "P4_HT_PO": { + "mean_abs_err": 29.735746023923575, + "std_abs_err": 25.12031739395562 + }, + "P4_LD": { + "mean_abs_err": 48.76021452336079, + "std_abs_err": 44.62470474768299 + }, + "P4_ST_FD": { + "mean_abs_err": 0.0005018763534501847, + "std_abs_err": 0.00014370042316070165 + }, + "P4_ST_GOV": { + "mean_abs_err": 2362.353038330857, + "std_abs_err": 1097.5596982548627 + }, + "P4_ST_LD": { + "mean_abs_err": 50.11671123231781, + "std_abs_err": 24.698663801562525 + }, + "P4_ST_PO": { + "mean_abs_err": 43.21710301636705, + "std_abs_err": 20.333811486356392 + }, + "P4_ST_PT01": { + "mean_abs_err": 3.8724539805389213, + "std_abs_err": 11.323945339634093 + }, + "P4_ST_TT01": { + "mean_abs_err": 12.006014205588144, + "std_abs_err": 4.292834677592534 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 1.0, + "P1_B2016": 0.7187333333333333, + "P1_B3004": 0.8108, + "P1_B3005": 0.8106, + "P1_B4002": 0.81055, + "P1_B4005": 0.1768, + "P1_B400B": 0.40025, + "P1_B4022": 0.5730333333333334, + "P1_FCV02Z": 0.83835, + "P1_FCV03D": 0.3987833333333334, + "P1_FCV03Z": 0.8114, + "P1_FT01": 0.23039166666666666, + "P1_FT01Z": 0.2356541666666666, + "P1_FT02": 0.33279166666666665, + "P1_FT02Z": 0.8362, + "P1_FT03": 0.8074, + "P1_FT03Z": 1.0, + "P1_LCV01D": 0.4262, + "P1_LCV01Z": 0.482725, + "P1_LIT01": 0.19719166666666665, + "P1_PCV01D": 0.6327083333333334, + "P1_PCV01Z": 0.5213041666666667, + "P1_PCV02Z": 1.0, + "P1_PIT01": 0.5208999999999999, + "P1_PIT02": 0.30900000000000005, + "P1_TIT01": 0.5214, + "P1_TIT02": 0.497, + "P2_24Vdc": 0.047587500000000005, + "P2_CO_rpm": 0.16000416666666661, + "P2_HILout": 0.22434583333333336, + "P2_MSD": 1.0, + "P2_SIT01": 0.62935, + "P2_SIT02": 0.74355, + "P2_VT01": 0.1963, + "P2_VXT02": 0.2699833333333333, + "P2_VXT03": 0.3601375, + "P2_VYT02": 0.30165, + "P2_VYT03": 0.366675, + "P3_FIT01": 0.6070791666666666, + "P3_LCP01D": 0.4425708333333333, + "P3_LCV01D": 0.52935, + "P3_LIT01": 0.45738749999999995, + "P3_PIT01": 0.42594583333333336, + "P4_HT_FD": 0.6169, + "P4_HT_LD": 0.9375, + "P4_HT_PO": 0.5689833333333334, + "P4_LD": 0.5931500000000001, + "P4_ST_FD": 0.13880416666666667, + "P4_ST_GOV": 0.7524958333333334, + "P4_ST_LD": 0.78055, + "P4_ST_PO": 0.7461916666666666, + "P4_ST_PT01": 0.5037625, + "P4_ST_TT01": 0.47985 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.08333, + "q25_diff": 0.012630000000000002, + "q50_diff": 0.012630000000000002, + "q75_diff": 0.012630000000000002, + "q95_diff": 0.012630000000000002 + }, + "P1_B2016": { + "q05_diff": 0.003290000000000015, + "q25_diff": 0.10759999999999992, + "q50_diff": 0.1572500000000001, + "q75_diff": 0.20982999999999996, + "q95_diff": 0.21500000000000008 + }, + "P1_B3004": { + "q05_diff": 2.8398500000000126, + "q25_diff": 1.6357399999999984, + "q50_diff": 1.6357399999999984, + "q75_diff": 1.6357399999999984, + "q95_diff": 1.6357399999999984 + }, + "P1_B3005": { + "q05_diff": 13.44177000000002, + "q25_diff": 13.44177000000002, + "q50_diff": 13.44177000000002, + "q75_diff": 13.44177000000002, + "q95_diff": 99.67999000000009 + }, + "P1_B4002": { + "q05_diff": 1.5437500000000028, + "q25_diff": 0.11175000000000068, + "q50_diff": 0.11175000000000068, + "q75_diff": 0.11175000000000068, + "q95_diff": 0.11175000000000068 + }, + "P1_B4005": { + "q05_diff": 99.54855, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 2314.8244999999997, + "q25_diff": 2.7055700000000797, + "q50_diff": 4.772950000000037, + "q75_diff": 6.878660000000309, + "q95_diff": 9.681150000000343 + }, + "P1_B4022": { + "q05_diff": 1.2294699999999992, + "q25_diff": 0.1574500000000043, + "q50_diff": 0.4875299999999996, + "q75_diff": 0.5946499999999943, + "q95_diff": 0.6461800000000011 + }, + "P1_FCV02Z": { + "q05_diff": 0.015260000000000051, + "q25_diff": 0.022890000000000077, + "q50_diff": 0.022890000000000077, + "q75_diff": 0.03815000000000013, + "q95_diff": 98.98715999999999 + }, + "P1_FCV03D": { + "q05_diff": 0.3449600000000004, + "q25_diff": 0.026730000000000587, + "q50_diff": 0.2911800000000042, + "q75_diff": 0.6531500000000037, + "q95_diff": 7.9212400000000045 + }, + "P1_FCV03Z": { + "q05_diff": 5.432130000000001, + "q25_diff": 5.254080000000002, + "q50_diff": 7.431929999999994, + "q75_diff": 7.330190000000002, + "q95_diff": 7.65227999999999 + }, + "P1_FT01": { + "q05_diff": 14.70313999999999, + "q25_diff": 2.42183, + "q50_diff": 1.5259300000000167, + "q75_diff": 4.196200000000005, + "q95_diff": 17.547579999999982 + }, + "P1_FT01Z": { + "q05_diff": 31.74590999999998, + "q25_diff": 6.19538, + "q50_diff": 3.443239999999946, + "q75_diff": 12.176449999999932, + "q95_diff": 45.661929999999984 + }, + "P1_FT02": { + "q05_diff": 800.3658399999999, + "q25_diff": 3.6347699999998895, + "q50_diff": 5.72180000000003, + "q75_diff": 8.333370000000059, + "q95_diff": 9.918340000000171 + }, + "P1_FT02Z": { + "q05_diff": 772.9251499999999, + "q25_diff": 865.57556, + "q50_diff": 865.2961499999999, + "q75_diff": 866.1954300000002, + "q95_diff": 865.6153600000002 + }, + "P1_FT03": { + "q05_diff": 11.449189999999987, + "q25_diff": 18.110030000000023, + "q50_diff": 43.34487999999999, + "q75_diff": 43.70285999999999, + "q95_diff": 17.708319999999958 + }, + "P1_FT03Z": { + "q05_diff": 741.28467, + "q25_diff": 736.08868, + "q50_diff": 712.07446, + "q75_diff": 713.6695, + "q95_diff": 819.81604 + }, + "P1_LCV01D": { + "q05_diff": 0.5801000000000007, + "q25_diff": 0.029520000000000657, + "q50_diff": 0.7149099999999997, + "q75_diff": 2.749790000000001, + "q95_diff": 3.27796 + }, + "P1_LCV01Z": { + "q05_diff": 0.4882900000000001, + "q25_diff": 0.0010599999999998388, + "q50_diff": 0.7018999999999993, + "q75_diff": 2.9525700000000006, + "q95_diff": 3.4408600000000007 + }, + "P1_LIT01": { + "q05_diff": 3.5096100000000092, + "q25_diff": 0.7156700000000455, + "q50_diff": 0.05198000000001457, + "q75_diff": 1.1245400000000245, + "q95_diff": 0.40893999999997277 + }, + "P1_PCV01D": { + "q05_diff": 1.1923600000000008, + "q25_diff": 2.8271599999999992, + "q50_diff": 5.671670000000006, + "q75_diff": 8.69249, + "q95_diff": 0.40302000000000504 + }, + "P1_PCV01Z": { + "q05_diff": 2.31015, + "q25_diff": 1.419070000000005, + "q50_diff": 3.6745599999999996, + "q75_diff": 7.064019999999999, + "q95_diff": 2.9428100000000015 + }, + "P1_PCV02Z": { + "q05_diff": 0.2212499999999995, + "q25_diff": 0.2212499999999995, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.2288800000000002, + "q95_diff": 0.2288800000000002 + }, + "P1_PIT01": { + "q05_diff": 0.10681000000000007, + "q25_diff": 0.02442000000000011, + "q50_diff": 0.016019999999999923, + "q75_diff": 0.05035000000000012, + "q95_diff": 0.10909999999999997 + }, + "P1_PIT02": { + "q05_diff": 0.07094999999999999, + "q25_diff": 0.0, + "q50_diff": 0.0007699999999999929, + "q75_diff": 0.0007600000000000384, + "q95_diff": 0.0015200000000000213 + }, + "P1_TIT01": { + "q05_diff": 0.10681000000000296, + "q25_diff": 0.15258000000000038, + "q50_diff": 0.5843500000000006, + "q75_diff": 0.9613000000000014, + "q95_diff": 1.1291399999999996 + }, + "P1_TIT02": { + "q05_diff": 0.015260000000004936, + "q25_diff": 0.32043999999999784, + "q50_diff": 0.9460399999999964, + "q75_diff": 1.4648400000000024, + "q95_diff": 3.0670199999999994 + }, + "P2_24Vdc": { + "q05_diff": 0.0008100000000013097, + "q25_diff": 0.0, + "q50_diff": 9.999999999976694e-05, + "q75_diff": 0.0, + "q95_diff": 0.000809999999997757 + }, + "P2_CO_rpm": { + "q05_diff": 1.4375, + "q25_diff": 1.0, + "q50_diff": 4.0, + "q75_diff": 7.0, + "q95_diff": 3.0 + }, + "P2_HILout": { + "q05_diff": 3.1946499999999105, + "q25_diff": 2.6348199999999906, + "q50_diff": 4.044309999999996, + "q75_diff": 4.874749999999949, + "q95_diff": 3.9745399999999336 + }, + "P2_MSD": { + "q05_diff": 4.750739999999951, + "q25_diff": 4.750739999999951, + "q50_diff": 4.750739999999951, + "q75_diff": 4.750739999999951, + "q95_diff": 4.750739999999951 + }, + "P2_SIT01": { + "q05_diff": 7.0, + "q25_diff": 6.0, + "q50_diff": 6.0, + "q75_diff": 6.0, + "q95_diff": 4.0 + }, + "P2_SIT02": { + "q05_diff": 8.653869999999984, + "q25_diff": 7.362730000000056, + "q50_diff": 7.191100000000006, + "q75_diff": 6.750490000000013, + "q95_diff": 5.111449999999991 + }, + "P2_VT01": { + "q05_diff": 0.021240000000000592, + "q25_diff": 0.0377799999999997, + "q50_diff": 0.008070000000000022, + "q75_diff": 0.010880000000000223, + "q95_diff": 0.012109999999999843 + }, + "P2_VXT02": { + "q05_diff": 0.25753000000000004, + "q25_diff": 0.34694999999999965, + "q50_diff": 0.20943999999999985, + "q75_diff": 0.019890000000000185, + "q95_diff": 0.21357999999999988 + }, + "P2_VXT03": { + "q05_diff": 0.23455999999999988, + "q25_diff": 0.30672999999999995, + "q50_diff": 0.24998999999999993, + "q75_diff": 0.13796000000000008, + "q95_diff": 0.19257999999999997 + }, + "P2_VYT02": { + "q05_diff": 0.31522000000000006, + "q25_diff": 0.40493999999999986, + "q50_diff": 0.21096000000000004, + "q75_diff": 0.1714500000000001, + "q95_diff": 0.3127000000000004 + }, + "P2_VYT03": { + "q05_diff": 0.27542000000000044, + "q25_diff": 0.34633999999999965, + "q50_diff": 0.16134999999999966, + "q75_diff": 0.1588099999999999, + "q95_diff": 0.29079999999999995 + }, + "P3_FIT01": { + "q05_diff": 2.0, + "q25_diff": 3.0, + "q50_diff": 74.0, + "q75_diff": 2682.0, + "q95_diff": 5032.0 + }, + "P3_LCP01D": { + "q05_diff": 1339.24536, + "q25_diff": 5602.49268, + "q50_diff": 8420.2207, + "q75_diff": 1168.3945299999996, + "q95_diff": 2130.4511700000003 + }, + "P3_LCV01D": { + "q05_diff": 320.0, + "q25_diff": 208.0, + "q50_diff": 8704.0, + "q75_diff": 13623.03394, + "q95_diff": 10454.587889999999 + }, + "P3_LIT01": { + "q05_diff": 3957.3466800000006, + "q25_diff": 883.5517600000003, + "q50_diff": 1176.5166000000008, + "q75_diff": 3351.8925799999997, + "q95_diff": 5827.252930000001 + }, + "P3_PIT01": { + "q05_diff": 2.0, + "q25_diff": 1.0, + "q50_diff": 1.0, + "q75_diff": 927.0, + "q95_diff": 3523.0 + }, + "P4_HT_FD": { + "q05_diff": 0.00152, + "q25_diff": 0.00058, + "q50_diff": 0.00087, + "q75_diff": 0.00116, + "q95_diff": 0.0002900000000000001 + }, + "P4_HT_LD": { + "q05_diff": 124.7862, + "q25_diff": 124.7862, + "q50_diff": 179.53157, + "q75_diff": 193.36982, + "q95_diff": 56.57957999999999 + }, + "P4_HT_PO": { + "q05_diff": 0.1043, + "q25_diff": 0.6148999999999999, + "q50_diff": 41.620250000000006, + "q75_diff": 62.12276, + "q95_diff": 61.017129999999995 + }, + "P4_LD": { + "q05_diff": 25.31896999999998, + "q25_diff": 15.10057999999998, + "q50_diff": 51.189790000000016, + "q75_diff": 89.02978999999999, + "q95_diff": 106.50228999999996 + }, + "P4_ST_FD": { + "q05_diff": 0.00014000000000000037, + "q25_diff": 0.00065, + "q50_diff": 0.00035999999999999997, + "q75_diff": 0.0005, + "q95_diff": 0.0008000000000000001 + }, + "P4_ST_GOV": { + "q05_diff": 383.3447300000007, + "q25_diff": 1649.9833999999992, + "q50_diff": 2242.03125, + "q75_diff": 3068.233399999999, + "q95_diff": 3115.007809999999 + }, + "P4_ST_LD": { + "q05_diff": 2.9424099999999953, + "q25_diff": 34.28836000000001, + "q50_diff": 48.83840999999995, + "q75_diff": 66.67095999999998, + "q95_diff": 72.90879000000001 + }, + "P4_ST_PO": { + "q05_diff": 6.265989999999988, + "q25_diff": 30.496280000000013, + "q50_diff": 41.687430000000006, + "q75_diff": 55.86928999999998, + "q95_diff": 57.018219999999985 + }, + "P4_ST_PT01": { + "q05_diff": 11.0, + "q25_diff": 13.0, + "q50_diff": 3.0, + "q75_diff": 1.0, + "q95_diff": 31.0 + }, + "P4_ST_TT01": { + "q05_diff": 17.0, + "q25_diff": 7.0, + "q50_diff": 30.0, + "q75_diff": 9.0, + "q95_diff": 2.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0189476811306442e-05, + "P1_B2016": 0.04553357314927986, + "P1_B3004": 1.018740504876714e-05, + "P1_B3005": 0.9999898125942444, + "P1_B4002": 0.9999898146591784, + "P1_B4005": 0.38925938845217534, + "P1_B400B": 0.8381901808238477, + "P1_B4022": 0.03037469907643997, + "P1_FCV02Z": 0.10069291191782703, + "P1_FCV03D": 0.0777684350647242, + "P1_FCV03Z": 0.17102411440731358, + "P1_FT01": 0.09773439250467486, + "P1_FT01Z": 0.06621322312073108, + "P1_FT02": 0.7991053599113773, + "P1_FT02Z": 0.7991603385212889, + "P1_FT03": 0.2873344106698136, + "P1_FT03Z": 0.2871973536252067, + "P1_LCV01D": 0.08615132202262399, + "P1_LCV01Z": 0.06423969042670852, + "P1_LIT01": 0.055595254094252367, + "P1_PCV01D": 0.1475711267187968, + "P1_PCV01Z": 0.08620226812780918, + "P1_PCV02Z": 0.5669400491575541, + "P1_PIT01": 0.04492160796015221, + "P1_PIT02": 0.2520800862470085, + "P1_TIT01": 0.03761284577066293, + "P1_TIT02": 0.0470828264328359, + "P2_24Vdc": 0.16217201179448734, + "P2_CO_rpm": 0.41252359027750285, + "P2_HILout": 0.18310870024680342, + "P2_MSD": 0.0, + "P2_SIT01": 0.35424038976166516, + "P2_SIT02": 0.4671348814210137, + "P2_VT01": 0.13135106075652725, + "P2_VXT02": 0.1486004530195939, + "P2_VXT03": 0.14340460978192937, + "P2_VYT02": 0.12437748903529589, + "P2_VYT03": 0.10991504800420204, + "P3_FIT01": 0.9302985891041546, + "P3_LCP01D": 0.03501492344185164, + "P3_LCV01D": 0.08108262114132148, + "P3_LIT01": 0.045951245838872334, + "P3_PIT01": 0.9722653711499396, + "P4_HT_FD": 0.06726751916497281, + "P4_HT_LD": 0.15670838558697175, + "P4_HT_PO": 0.4735802135572973, + "P4_LD": 0.07837920481609106, + "P4_ST_FD": 0.1706154777306946, + "P4_ST_GOV": 0.05945532353994332, + "P4_ST_LD": 0.09636373379999075, + "P4_ST_PO": 0.055104496881990817, + "P4_ST_PT01": 0.546803788523067, + "P4_ST_TT01": 0.03649396378380221 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.44960440692017306, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + }, + "avg_ks": 0.5430239779874214, + "avg_jsd": 0.024122590895914533, + "avg_lag1_diff": 0.25321129348110133 +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_post_calibration__seed1337/eval_post.json b/example/results/ablations/runs/no_post_calibration__seed1337/eval_post.json new file mode 100644 index 0000000..bec2136 --- /dev/null +++ b/example/results/ablations/runs/no_post_calibration__seed1337/eval_post.json @@ -0,0 +1,971 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09355486979166669, + "std": 0.01799194293084728 + }, + "P1_B2016": { + "mean": 1.226465963541668, + "std": 0.05762137409951418 + }, + "P1_B3004": { + "mean": 403.2905014583335, + "std": 14.217563527951576 + }, + "P1_B3005": { + "mean": 1059.5253898958326, + "std": 58.42587703751758 + }, + "P1_B4002": { + "mean": 32.37490617187499, + "std": 0.6519533213854061 + }, + "P1_B4005": { + "mean": 64.99571291666655, + "std": 42.03467331899641 + }, + "P1_B400B": { + "mean": 2770.5973602343743, + "std": 241.08970370015072 + }, + "P1_B4022": { + "mean": 36.08671476562498, + "std": 0.7147782383668128 + }, + "P1_FCV02Z": { + "mean": 21.954832630208315, + "std": 38.87335152465215 + }, + "P1_FCV03D": { + "mean": 58.14541601562505, + "std": 6.267267768996455 + }, + "P1_FCV03Z": { + "mean": 58.21398343750003, + "std": 6.423211358701469 + }, + "P1_FT01": { + "mean": 173.78418085937486, + "std": 10.871636774164493 + }, + "P1_FT01Z": { + "mean": 832.3371609895837, + "std": 21.14366010462666 + }, + "P1_FT02": { + "mean": 1856.0462919531253, + "std": 368.80879559858437 + }, + "P1_FT02Z": { + "mean": 1843.7261292187509, + "std": 1190.4241853713818 + }, + "P1_FT03": { + "mean": 280.8603020833332, + "std": 14.266500816305975 + }, + "P1_FT03Z": { + "mean": 1058.2564329947922, + "std": 59.50957255113764 + }, + "P1_LCV01D": { + "mean": 11.699055963541667, + "std": 3.0155193220433567 + }, + "P1_LCV01Z": { + "mean": 11.627510442708333, + "std": 3.0251072332459787 + }, + "P1_LIT01": { + "mean": 398.16104973958295, + "std": 2.558201454443084 + }, + "P1_PCV01D": { + "mean": 65.64593611979173, + "std": 23.56629129628467 + }, + "P1_PCV01Z": { + "mean": 65.565628671875, + "std": 23.790956943468952 + }, + "P1_PCV02Z": { + "mean": 11.867279765625007, + "std": 0.10811550492611105 + }, + "P1_PIT01": { + "mean": 1.3642693489583342, + "std": 0.0045464753907211814 + }, + "P1_PIT02": { + "mean": 0.27590526041666674, + "std": 0.0003986936041241994 + }, + "P1_TIT01": { + "mean": 35.183942265625014, + "std": 0.25176049384259186 + }, + "P1_TIT02": { + "mean": 35.410734296875, + "std": 0.34275359801312044 + }, + "P2_24Vdc": { + "mean": 28.02860809895833, + "std": 0.0032970984602375326 + }, + "P2_CO_rpm": { + "mean": 54109.52951054688, + "std": 21.97475672644607 + }, + "P2_HILout": { + "mean": 711.4986896354167, + "std": 7.856828438379631 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 784.7902604166661, + "std": 3.249294028147303 + }, + "P2_SIT02": { + "mean": 785.8944788281248, + "std": 2.817230183482184 + }, + "P2_VT01": { + "mean": 11.922679921875003, + "std": 0.05504070160484108 + }, + "P2_VXT02": { + "mean": -3.3796592708333324, + "std": 0.3392395874187618 + }, + "P2_VXT03": { + "mean": -1.396118854166667, + "std": 0.18398561380285178 + }, + "P2_VYT02": { + "mean": 4.004463072916667, + "std": 0.389492910854868 + }, + "P2_VYT03": { + "mean": 6.30108776041666, + "std": 0.24693428433774522 + }, + "P3_FIT01": { + "mean": -19.732960494791662, + "std": 16.706026566947937 + }, + "P3_LCP01D": { + "mean": 8243.110082057286, + "std": 3711.12057106618 + }, + "P3_LCV01D": { + "mean": 1076.3946400000002, + "std": 1690.1832719119786 + }, + "P3_LIT01": { + "mean": 11849.491632994792, + "std": 1037.1359212752477 + }, + "P3_PIT01": { + "mean": -17.142703098958336, + "std": 3.7681811558923877 + }, + "P4_HT_FD": { + "mean": 0.000783671875, + "std": 0.0007698905056872621 + }, + "P4_HT_LD": { + "mean": 34.231568880208286, + "std": 33.60361854009684 + }, + "P4_HT_PO": { + "mean": 35.25451283854166, + "std": 31.650898099038645 + }, + "P4_LD": { + "mean": 317.36853927083354, + "std": 15.991101621760016 + }, + "P4_ST_FD": { + "mean": -0.0005309895833333335, + "std": 0.0014984758348293276 + }, + "P4_ST_GOV": { + "mean": 15550.608222916668, + "std": 754.5835177329105 + }, + "P4_ST_LD": { + "mean": 281.8861859374999, + "std": 13.154505490055021 + }, + "P4_ST_PO": { + "mean": 289.0518683072917, + "std": 14.117132016051908 + }, + "P4_ST_PT01": { + "mean": 10046.702702838544, + "std": 6.4608246856850355 + }, + "P4_ST_TT01": { + "mean": 27597.026677473943, + "std": 19.624487126975794 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 1.2782299308983225e-05, + "std_abs_err": 6.494151245123314e-06 + }, + "P1_B2016": { + "mean_abs_err": 0.16480036112309904, + "std_abs_err": 0.0781521883770489 + }, + "P1_B3004": { + "mean_abs_err": 0.09556236619266656, + "std_abs_err": 1.2005860600902345 + }, + "P1_B3005": { + "mean_abs_err": 1.5233966327971302, + "std_abs_err": 2.757439473482698 + }, + "P1_B4002": { + "mean_abs_err": 0.05797985199510691, + "std_abs_err": 0.07327982218632823 + }, + "P1_B4005": { + "mean_abs_err": 4.470605900468627, + "std_abs_err": 0.14662412871327746 + }, + "P1_B400B": { + "mean_abs_err": 992.1214900564062, + "std_abs_err": 952.262569218922 + }, + "P1_B4022": { + "mean_abs_err": 0.02554011822245883, + "std_abs_err": 0.04332738353199439 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.996102270263961, + "std_abs_err": 1.5651339146060295 + }, + "P1_FCV03D": { + "mean_abs_err": 0.42244743522177686, + "std_abs_err": 0.3322279497122995 + }, + "P1_FCV03Z": { + "mean_abs_err": 1.0850557426898035, + "std_abs_err": 0.29159260563013323 + }, + "P1_FT01": { + "mean_abs_err": 20.939473496905407, + "std_abs_err": 16.476345064479833 + }, + "P1_FT01Z": { + "mean_abs_err": 46.71252051456884, + "std_abs_err": 54.622042858691586 + }, + "P1_FT02": { + "mean_abs_err": 723.2281387184992, + "std_abs_err": 520.3873682528283 + }, + "P1_FT02Z": { + "mean_abs_err": 65.24768261010217, + "std_abs_err": 2.9211703997677887 + }, + "P1_FT03": { + "mean_abs_err": 1.7786681928957364, + "std_abs_err": 19.20771229498953 + }, + "P1_FT03Z": { + "mean_abs_err": 2.785605353729352, + "std_abs_err": 1.6159163124796905 + }, + "P1_LCV01D": { + "mean_abs_err": 0.45527669823346884, + "std_abs_err": 0.1009043690132354 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.33856358662958463, + "std_abs_err": 0.061637735900667234 + }, + "P1_LIT01": { + "mean_abs_err": 5.953822423423333, + "std_abs_err": 13.073696593930887 + }, + "P1_PCV01D": { + "mean_abs_err": 1.8196376158996515, + "std_abs_err": 0.20347540462090308 + }, + "P1_PCV01Z": { + "mean_abs_err": 2.842523125027583, + "std_abs_err": 0.07364286719469249 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.024526819020080026, + "std_abs_err": 0.011271039461463683 + }, + "P1_PIT01": { + "mean_abs_err": 0.013211871795057784, + "std_abs_err": 0.05688608397916092 + }, + "P1_PIT02": { + "mean_abs_err": 0.21125182992484998, + "std_abs_err": 0.49831294445075563 + }, + "P1_TIT01": { + "mean_abs_err": 0.49726597680820106, + "std_abs_err": 0.3315201333433571 + }, + "P1_TIT02": { + "mean_abs_err": 1.114244381461532, + "std_abs_err": 0.8790051280111426 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00010160366933931186, + "std_abs_err": 4.536765416938244e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 6.819312664207246, + "std_abs_err": 0.48097504834229454 + }, + "P2_HILout": { + "mean_abs_err": 0.035880365320508645, + "std_abs_err": 0.5978974388765312 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.8863430032231463, + "std_abs_err": 1.949621332296922 + }, + "P2_SIT02": { + "mean_abs_err": 1.9907409455785228, + "std_abs_err": 2.3733495733307826 + }, + "P2_VT01": { + "mean_abs_err": 0.0011357241351301894, + "std_abs_err": 0.013472980467160986 + }, + "P2_VXT02": { + "mean_abs_err": 0.07948079844333611, + "std_abs_err": 0.13179333885245847 + }, + "P2_VXT03": { + "mean_abs_err": 0.08922556750198907, + "std_abs_err": 0.16564176536080188 + }, + "P2_VYT02": { + "mean_abs_err": 0.02299422292372988, + "std_abs_err": 0.11041754213791988 + }, + "P2_VYT03": { + "mean_abs_err": 0.06668264761877563, + "std_abs_err": 0.13286867763334637 + }, + "P3_FIT01": { + "mean_abs_err": 1170.633136600975, + "std_abs_err": 1763.018671223327 + }, + "P3_LCP01D": { + "mean_abs_err": 3658.3829338167125, + "std_abs_err": 1418.8794266496984 + }, + "P3_LCV01D": { + "mean_abs_err": 6057.482076981241, + "std_abs_err": 5149.744623903843 + }, + "P3_LIT01": { + "mean_abs_err": 1505.3528596991946, + "std_abs_err": 3317.0387156526886 + }, + "P3_PIT01": { + "mean_abs_err": 666.5006061222861, + "std_abs_err": 1147.153749551303 + }, + "P4_HT_FD": { + "mean_abs_err": 0.0008868598203517398, + "std_abs_err": 0.0012621278087168383 + }, + "P4_HT_LD": { + "mean_abs_err": 0.23665180052267232, + "std_abs_err": 0.17982503009853446 + }, + "P4_HT_PO": { + "mean_abs_err": 1.2707050437847514, + "std_abs_err": 0.21002688171716954 + }, + "P4_LD": { + "mean_abs_err": 48.76021452336079, + "std_abs_err": 44.62470474768299 + }, + "P4_ST_FD": { + "mean_abs_err": 0.0005018763534501847, + "std_abs_err": 0.00014370042316070165 + }, + "P4_ST_GOV": { + "mean_abs_err": 2362.353038330857, + "std_abs_err": 1097.5596982548627 + }, + "P4_ST_LD": { + "mean_abs_err": 50.11671123231781, + "std_abs_err": 24.698663801562525 + }, + "P4_ST_PO": { + "mean_abs_err": 43.21710301636705, + "std_abs_err": 20.333811486356392 + }, + "P4_ST_PT01": { + "mean_abs_err": 3.8724539805389213, + "std_abs_err": 11.323945339634093 + }, + "P4_ST_TT01": { + "mean_abs_err": 12.006014205588144, + "std_abs_err": 4.292834677592534 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.6595583333333334, + "P1_B2016": 0.7187333333333333, + "P1_B3004": 0.671875, + "P1_B3005": 0.5293, + "P1_B4002": 0.7923208333333334, + "P1_B4005": 0.3062625, + "P1_B400B": 0.40025, + "P1_B4022": 0.5756333333333334, + "P1_FCV02Z": 0.7746, + "P1_FCV03D": 0.45162500000000005, + "P1_FCV03Z": 0.40985833333333327, + "P1_FT01": 0.23039166666666666, + "P1_FT01Z": 0.2356541666666666, + "P1_FT02": 0.33279166666666665, + "P1_FT02Z": 0.4029958333333334, + "P1_FT03": 0.8074, + "P1_FT03Z": 0.46950833333333336, + "P1_LCV01D": 0.3859916666666667, + "P1_LCV01Z": 0.397625, + "P1_LIT01": 0.19719166666666665, + "P1_PCV01D": 0.5304583333333333, + "P1_PCV01Z": 0.5184416666666667, + "P1_PCV02Z": 0.6588541666666666, + "P1_PIT01": 0.5208999999999999, + "P1_PIT02": 0.30900000000000005, + "P1_TIT01": 0.5214, + "P1_TIT02": 0.497, + "P2_24Vdc": 0.13548333333333334, + "P2_CO_rpm": 0.16000416666666661, + "P2_HILout": 0.08242499999999997, + "P2_MSD": 0.0, + "P2_SIT01": 0.62935, + "P2_SIT02": 0.74355, + "P2_VT01": 0.1963, + "P2_VXT02": 0.2699833333333333, + "P2_VXT03": 0.3601375, + "P2_VYT02": 0.30165, + "P2_VYT03": 0.366675, + "P3_FIT01": 0.6070791666666666, + "P3_LCP01D": 0.4425708333333333, + "P3_LCV01D": 0.52935, + "P3_LIT01": 0.45738749999999995, + "P3_PIT01": 0.42594583333333336, + "P4_HT_FD": 0.6169, + "P4_HT_LD": 0.06742500000000001, + "P4_HT_PO": 0.05970416666666667, + "P4_LD": 0.5931500000000001, + "P4_ST_FD": 0.13880416666666667, + "P4_ST_GOV": 0.7524958333333334, + "P4_ST_LD": 0.78055, + "P4_ST_PO": 0.7461916666666666, + "P4_ST_PT01": 0.5037625, + "P4_ST_TT01": 0.47985 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.02845, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0011099999999999999, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.003290000000000015, + "q25_diff": 0.10759999999999992, + "q50_diff": 0.1572500000000001, + "q75_diff": 0.20982999999999996, + "q95_diff": 0.21500000000000008 + }, + "P1_B3004": { + "q05_diff": 14.80313000000001, + "q25_diff": 1.6197799999999916, + "q50_diff": 6.726189999999974, + "q75_diff": 8.508819999999957, + "q95_diff": 37.86444 + }, + "P1_B3005": { + "q05_diff": 40.40789999999993, + "q25_diff": 8.045049999999947, + "q50_diff": 89.51202, + "q75_diff": 104.97393, + "q95_diff": 4.4593499999998585 + }, + "P1_B4002": { + "q05_diff": 0.4564699999999995, + "q25_diff": 1.6555000000000035, + "q50_diff": 1.6555000000000035, + "q75_diff": 0.9287000000000063, + "q95_diff": 0.16969000000000278 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 81.96193, + "q50_diff": 0.04000000000000625, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 2314.8244999999997, + "q25_diff": 2.7055700000000797, + "q50_diff": 4.772950000000037, + "q75_diff": 6.878660000000309, + "q95_diff": 9.681150000000343 + }, + "P1_B4022": { + "q05_diff": 0.38222999999999985, + "q25_diff": 1.3479000000000028, + "q50_diff": 1.4273799999999994, + "q75_diff": 0.9127399999999994, + "q95_diff": 0.5270299999999963 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.05340999999999996, + "q75_diff": 38.169869999999996, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 1.3654200000000003, + "q25_diff": 0.2554099999999977, + "q50_diff": 8.60333, + "q75_diff": 10.84863, + "q95_diff": 0.824300000000008 + }, + "P1_FCV03Z": { + "q05_diff": 1.7547599999999974, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.148199999999996, + "q75_diff": 10.864249999999998, + "q95_diff": 0.9307899999999876 + }, + "P1_FT01": { + "q05_diff": 14.70313999999999, + "q25_diff": 2.42183, + "q50_diff": 1.5259300000000167, + "q75_diff": 4.196200000000005, + "q95_diff": 17.547579999999982 + }, + "P1_FT01Z": { + "q05_diff": 31.74590999999998, + "q25_diff": 6.19538, + "q50_diff": 3.443239999999946, + "q75_diff": 12.176449999999932, + "q95_diff": 45.661929999999984 + }, + "P1_FT02": { + "q05_diff": 800.3658399999999, + "q25_diff": 3.6347699999998895, + "q50_diff": 5.72180000000003, + "q75_diff": 8.333370000000059, + "q95_diff": 9.918340000000171 + }, + "P1_FT02Z": { + "q05_diff": 1.9862300000000026, + "q25_diff": 2300.54303, + "q50_diff": 14.318849999999657, + "q75_diff": 7.299800000000232, + "q95_diff": 6.317140000000109 + }, + "P1_FT03": { + "q05_diff": 11.449189999999987, + "q25_diff": 18.110030000000023, + "q50_diff": 43.34487999999999, + "q75_diff": 43.70285999999999, + "q95_diff": 17.708319999999958 + }, + "P1_FT03Z": { + "q05_diff": 23.41980000000001, + "q25_diff": 2.9805299999999306, + "q50_diff": 84.98266000000012, + "q75_diff": 100.6383699999999, + "q95_diff": 3.5079399999999623 + }, + "P1_LCV01D": { + "q05_diff": 0.17497000000000007, + "q25_diff": 0.29953000000000074, + "q50_diff": 4.171619999999999, + "q75_diff": 3.162939999999999, + "q95_diff": 0.7125800000000009 + }, + "P1_LCV01Z": { + "q05_diff": 0.0, + "q25_diff": 0.5874600000000001, + "q50_diff": 4.638670000000001, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.9536700000000007 + }, + "P1_LIT01": { + "q05_diff": 3.5096100000000092, + "q25_diff": 0.7156700000000455, + "q50_diff": 0.05198000000001457, + "q75_diff": 1.1245400000000245, + "q95_diff": 0.40893999999997277 + }, + "P1_PCV01D": { + "q05_diff": 0.18625000000000114, + "q25_diff": 2.7490099999999984, + "q50_diff": 18.274499999999996, + "q75_diff": 43.266600000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.22888000000000375, + "q25_diff": 2.0370500000000007, + "q50_diff": 16.029360000000004, + "q75_diff": 43.33495, + "q95_diff": 45.1493 + }, + "P1_PCV02Z": { + "q05_diff": 0.2288800000000002, + "q25_diff": 0.2212499999999995, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_PIT01": { + "q05_diff": 0.10681000000000007, + "q25_diff": 0.02442000000000011, + "q50_diff": 0.016019999999999923, + "q75_diff": 0.05035000000000012, + "q95_diff": 0.10909999999999997 + }, + "P1_PIT02": { + "q05_diff": 0.07094999999999999, + "q25_diff": 0.0, + "q50_diff": 0.0007699999999999929, + "q75_diff": 0.0007600000000000384, + "q95_diff": 0.0015200000000000213 + }, + "P1_TIT01": { + "q05_diff": 0.10681000000000296, + "q25_diff": 0.15258000000000038, + "q50_diff": 0.5843500000000006, + "q75_diff": 0.9613000000000014, + "q95_diff": 1.1291399999999996 + }, + "P1_TIT02": { + "q05_diff": 0.015260000000004936, + "q25_diff": 0.32043999999999784, + "q50_diff": 0.9460399999999964, + "q75_diff": 1.4648400000000024, + "q95_diff": 3.0670199999999994 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.00013999999999825263, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.00010999999999938836 + }, + "P2_CO_rpm": { + "q05_diff": 1.4375, + "q25_diff": 1.0, + "q50_diff": 4.0, + "q75_diff": 7.0, + "q95_diff": 3.0 + }, + "P2_HILout": { + "q05_diff": 0.3295900000000529, + "q25_diff": 1.6540499999999838, + "q50_diff": 0.31737999999995736, + "q75_diff": 1.7334000000000742, + "q95_diff": 2.2399900000000343 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 7.0, + "q25_diff": 6.0, + "q50_diff": 6.0, + "q75_diff": 6.0, + "q95_diff": 4.0 + }, + "P2_SIT02": { + "q05_diff": 8.653869999999984, + "q25_diff": 7.362730000000056, + "q50_diff": 7.191100000000006, + "q75_diff": 6.750490000000013, + "q95_diff": 5.111449999999991 + }, + "P2_VT01": { + "q05_diff": 0.021240000000000592, + "q25_diff": 0.0377799999999997, + "q50_diff": 0.008070000000000022, + "q75_diff": 0.010880000000000223, + "q95_diff": 0.012109999999999843 + }, + "P2_VXT02": { + "q05_diff": 0.25753000000000004, + "q25_diff": 0.34694999999999965, + "q50_diff": 0.20943999999999985, + "q75_diff": 0.019890000000000185, + "q95_diff": 0.21357999999999988 + }, + "P2_VXT03": { + "q05_diff": 0.23455999999999988, + "q25_diff": 0.30672999999999995, + "q50_diff": 0.24998999999999993, + "q75_diff": 0.13796000000000008, + "q95_diff": 0.19257999999999997 + }, + "P2_VYT02": { + "q05_diff": 0.31522000000000006, + "q25_diff": 0.40493999999999986, + "q50_diff": 0.21096000000000004, + "q75_diff": 0.1714500000000001, + "q95_diff": 0.3127000000000004 + }, + "P2_VYT03": { + "q05_diff": 0.27542000000000044, + "q25_diff": 0.34633999999999965, + "q50_diff": 0.16134999999999966, + "q75_diff": 0.1588099999999999, + "q95_diff": 0.29079999999999995 + }, + "P3_FIT01": { + "q05_diff": 2.0, + "q25_diff": 3.0, + "q50_diff": 74.0, + "q75_diff": 2682.0, + "q95_diff": 5032.0 + }, + "P3_LCP01D": { + "q05_diff": 1339.24536, + "q25_diff": 5602.49268, + "q50_diff": 8420.2207, + "q75_diff": 1168.3945299999996, + "q95_diff": 2130.4511700000003 + }, + "P3_LCV01D": { + "q05_diff": 320.0, + "q25_diff": 208.0, + "q50_diff": 8704.0, + "q75_diff": 13623.03394, + "q95_diff": 10454.587889999999 + }, + "P3_LIT01": { + "q05_diff": 3957.3466800000006, + "q25_diff": 883.5517600000003, + "q50_diff": 1176.5166000000008, + "q75_diff": 3351.8925799999997, + "q95_diff": 5827.252930000001 + }, + "P3_PIT01": { + "q05_diff": 2.0, + "q25_diff": 1.0, + "q50_diff": 1.0, + "q75_diff": 927.0, + "q95_diff": 3523.0 + }, + "P4_HT_FD": { + "q05_diff": 0.00152, + "q25_diff": 0.00058, + "q50_diff": 0.00087, + "q75_diff": 0.00116, + "q95_diff": 0.0002900000000000001 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 27.14122, + "q75_diff": 0.13020999999999106, + "q95_diff": 0.21701000000000192 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 0.9223299999999999, + "q50_diff": 8.89752, + "q75_diff": 0.054199999999994475, + "q95_diff": 0.6510700000000043 + }, + "P4_LD": { + "q05_diff": 25.31896999999998, + "q25_diff": 15.10057999999998, + "q50_diff": 51.189790000000016, + "q75_diff": 89.02978999999999, + "q95_diff": 106.50228999999996 + }, + "P4_ST_FD": { + "q05_diff": 0.00014000000000000037, + "q25_diff": 0.00065, + "q50_diff": 0.00035999999999999997, + "q75_diff": 0.0005, + "q95_diff": 0.0008000000000000001 + }, + "P4_ST_GOV": { + "q05_diff": 383.3447300000007, + "q25_diff": 1649.9833999999992, + "q50_diff": 2242.03125, + "q75_diff": 3068.233399999999, + "q95_diff": 3115.007809999999 + }, + "P4_ST_LD": { + "q05_diff": 2.9424099999999953, + "q25_diff": 34.28836000000001, + "q50_diff": 48.83840999999995, + "q75_diff": 66.67095999999998, + "q95_diff": 72.90879000000001 + }, + "P4_ST_PO": { + "q05_diff": 6.265989999999988, + "q25_diff": 30.496280000000013, + "q50_diff": 41.687430000000006, + "q75_diff": 55.86928999999998, + "q95_diff": 57.018219999999985 + }, + "P4_ST_PT01": { + "q05_diff": 11.0, + "q25_diff": 13.0, + "q50_diff": 3.0, + "q75_diff": 1.0, + "q95_diff": 31.0 + }, + "P4_ST_TT01": { + "q05_diff": 17.0, + "q25_diff": 7.0, + "q50_diff": 30.0, + "q75_diff": 9.0, + "q95_diff": 2.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0624831607922858, + "P1_B2016": 0.04553357314927986, + "P1_B3004": 0.9420514424699565, + "P1_B3005": 0.9755864990985265, + "P1_B4002": 0.9939428775878422, + "P1_B4005": 1.0901635813805555, + "P1_B400B": 0.8381901808238477, + "P1_B4022": 1.0728287547128275, + "P1_FCV02Z": 1.0856923346443097, + "P1_FCV03D": 1.0287616800381814, + "P1_FCV03Z": 1.01920928123416, + "P1_FT01": 0.09773439250467486, + "P1_FT01Z": 0.06621322312073108, + "P1_FT02": 0.7991053599113773, + "P1_FT02Z": 1.0572315062387077, + "P1_FT03": 0.2873344106698136, + "P1_FT03Z": 0.9861265787637634, + "P1_LCV01D": 0.9922897795117445, + "P1_LCV01Z": 0.9035160861736835, + "P1_LIT01": 0.055595254094252367, + "P1_PCV01D": 0.9847169107083952, + "P1_PCV01Z": 1.1483081333334428, + "P1_PCV02Z": 0.5764046458797007, + "P1_PIT01": 0.04492160796015221, + "P1_PIT02": 0.2520800862470085, + "P1_TIT01": 0.03761284577066293, + "P1_TIT02": 0.0470828264328359, + "P2_24Vdc": 0.039965792976989475, + "P2_CO_rpm": 0.41252359027750285, + "P2_HILout": 0.24706059027478758, + "P2_MSD": 0.0, + "P2_SIT01": 0.35424038976166516, + "P2_SIT02": 0.4671348814210137, + "P2_VT01": 0.13135106075652725, + "P2_VXT02": 0.1486004530195939, + "P2_VXT03": 0.14340460978192937, + "P2_VYT02": 0.12437748903529589, + "P2_VYT03": 0.10991504800420204, + "P3_FIT01": 0.9302985891041546, + "P3_LCP01D": 0.03501492344185164, + "P3_LCV01D": 0.08108262114132148, + "P3_LIT01": 0.045951245838872334, + "P3_PIT01": 0.9722653711499396, + "P4_HT_FD": 0.06726751916497281, + "P4_HT_LD": 1.052071148480924, + "P4_HT_PO": 1.0918064404320895, + "P4_LD": 0.07837920481609106, + "P4_ST_FD": 0.1706154777306946, + "P4_ST_GOV": 0.05945532353994332, + "P4_ST_LD": 0.09636373379999075, + "P4_ST_PO": 0.055104496881990817, + "P4_ST_PT01": 0.546803788523067, + "P4_ST_TT01": 0.03649396378380221 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.44960440692017306, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_quantile__seed1337/comprehensive_eval.json b/example/results/ablations/runs/no_quantile__seed1337/comprehensive_eval.json new file mode 100644 index 0000000..c913e6f --- /dev/null +++ b/example/results/ablations/runs/no_quantile__seed1337/comprehensive_eval.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_quantile__seed1337/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.5986268081761006, + "avg_jsd": 0.010030341725636, + "avg_lag1_diff": 0.15569708276765754 + }, + "two_sample": { + "continuous_mmd_rbf": 0.7526681423187256, + "continuous_mmd_gamma": 1.0540935269187076e-05, + "histogram_mmd_rbf": 0.7526563405990601, + "histogram_mmd_gamma": 1.05401766418322e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 170.52767944335938, + "nn_holdout_to_train_mean": 70.52462005615234, + "memorization_ratio": 2.4179879212051576, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.30008164048194885, + "corr_frobenius": 21.361055374145508, + "lag1_corr_mean_abs_diff": 0.29954877495765686, + "lag1_corr_frobenius": 21.304372787475586, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.3157265782356262, + "corr_frobenius": 11.02734088897705, + "lag1_corr_mean_abs_diff": 0.3185610771179199, + "lag1_corr_frobenius": 11.136503219604492 + }, + "P2": { + "corr_mean_abs_diff": 0.2017672210931778, + "corr_frobenius": 3.9136135578155518, + "lag1_corr_mean_abs_diff": 0.17747682332992554, + "lag1_corr_frobenius": 3.1356148719787598 + }, + "P3": { + "corr_mean_abs_diff": 0.29451829195022583, + "corr_frobenius": 2.0890676975250244, + "lag1_corr_mean_abs_diff": 0.3015052080154419, + "lag1_corr_frobenius": 2.1081626415252686 + }, + "P4": { + "corr_mean_abs_diff": 0.4004734456539154, + "corr_frobenius": 5.465811729431152, + "lag1_corr_mean_abs_diff": 0.41377076506614685, + "lag1_corr_frobenius": 5.516853332519531 + } + } + }, + "frequency": { + "avg_psd_l1": 0.0160636268556118, + "avg_psd_cosine": 0.30595176223053333, + "avg_low_high_ratio_abs_diff": 5660491.196250632 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 15.0, + "mean_dwell": 24.0, + "median_dwell": 2.5, + "mean_step": 0.0337586657765011, + "median_step": 0.021709997206926346 + }, + "P1_B2004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B3004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.013833596251963324, + "median_step": 0.009159088134765625 + }, + "P1_B3005": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22998.14285714286, + "mean_abs_diff_mean_dwell": 14201.625685203559, + "mean_abs_diff_median_dwell": 14207.642857142857, + "mean_abs_diff_mean_step": 0.5046434971487197, + "mean_abs_diff_median_step": 0.11230059526860714 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.7369791666666666, + "change_rate": 1.0, + "step_median": 1.5560455322265625 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.14802042643229163, + "mean_abs_diff_change_rate": 0.5391798826078553, + "mean_abs_diff_step_median": 1.5240252315998077 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.010416666666666666, + "top1_mass": 0.921875, + "top3_mass": 0.9921875, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.4895833333333333, + "top1_mass": 0.5, + "top3_mass": 0.5104166666666666, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.4947916666666667, + "top1_mass": 0.4895833333333333, + "top3_mass": 0.5, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.9375, + "top1_mass": 0.0078125, + "top3_mass": 0.018229166666666668, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.828125, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.7630208333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.03125, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.5338541666666666, + "top1_mass": 0.015625, + "top3_mass": 0.046875, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.5520833333333334, + "top1_mass": 0.015625, + "top3_mass": 0.046875, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5443458557128906, + "mean_abs_diff_top1_mass": 0.21151097615559894, + "mean_abs_diff_top3_mass": 0.19818751017252603, + "mean_abs_diff_median_dwell": 0.0 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": -0.10307999700307846, + "q50": 0.3064199984073639, + "q95": 0.5731499791145325, + "tail_ratio": 0.6513552715423715 + }, + "P2_SIT02": { + "q05": 779.8792114257812, + "q50": 785.1359252929688, + "q95": 789.8136596679688, + "tail_ratio": 0.889859043726633 + }, + "P1_FT03": { + "q05": 78.47207641601562, + "q50": 117.61963653564453, + "q95": 249.82940673828125, + "tail_ratio": 3.377216097213314 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 43.93644383301338, + "mean_abs_diff_q50": 62.870752066373825, + "mean_abs_diff_q95": 23.31601144870122, + "mean_abs_diff_tail_ratio": 8.098831336922286 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 379.0, + "mean_dwell": 1.0105263157894737, + "median_dwell": 1.0, + "mean_step": 5.128579293203228, + "median_step": 2.242828369140625 + }, + "P1_FT02Z": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 60.14884355111782, + "median_step": 25.625 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89281.5, + "mean_abs_diff_mean_dwell": 0.09566734243752206, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 24.987610846645115, + "mean_abs_diff_median_step": 10.667007446289062 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 33.037071228027344, + "std": 12.766562461853027, + "lag1": 0.9784713983535767 + }, + "P2_24Vdc": { + "mean": 28.029876708984375, + "std": 0.003262430662289262, + "lag1": 0.1120024099946022 + }, + "P2_HILout": { + "mean": 711.7379760742188, + "std": 10.349284172058105, + "lag1": 0.04841964691877365 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.5195471445719401, + "mean_abs_diff_std": 6.888279523234814, + "mean_abs_diff_lag1": 0.09533786401152611 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.8651301860809326, + "mae": 0.4152381420135498 + }, + "synthetic_only": { + "rmse": 1.6529492139816284, + "mae": 1.0005885362625122 + }, + "real_plus_synthetic": { + "rmse": 1.013792634010315, + "mae": 0.4782671332359314 + }, + "rmse_ratio_synth_to_real": 1.9106363881134945 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8564471243042671, + "auprc": 0.4132698022160485, + "threshold": 0.3127160668373108, + "f1": 0.21333333333333335, + "precision": 0.125, + "recall": 0.7272727272727273, + "best_f1": 0.5684210526315789, + "best_f1_threshold": 0.6349313259124756 + }, + "synthetic_only": { + "auroc": 0.7455241187384045, + "auprc": 0.3300653727950748, + "threshold": 0.026501694694161415, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.4470588235294118, + "best_f1_threshold": 4.122805118560791 + }, + "real_plus_synthetic": { + "auroc": 0.8571196660482374, + "auprc": 0.4166041816171082, + "threshold": 0.31336838006973267, + "f1": 0.19701492537313434, + "precision": 0.1134020618556701, + "recall": 0.75, + "best_f1": 0.5684210526315789, + "best_f1_threshold": 0.6686344742774963 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_quantile__seed1337/comprehensive_eval_post.json b/example/results/ablations/runs/no_quantile__seed1337/comprehensive_eval_post.json new file mode 100644 index 0000000..7e71c2c --- /dev/null +++ b/example/results/ablations/runs/no_quantile__seed1337/comprehensive_eval_post.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_quantile__seed1337/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.562717080116272, + "continuous_mmd_gamma": 3.302989184568352e-06, + "histogram_mmd_rbf": 0.5627133846282959, + "histogram_mmd_gamma": 3.3028882725791118e-06, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 386.08416748046875, + "nn_holdout_to_train_mean": 70.52462005615234, + "memorization_ratio": 5.474459375648745, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.23569586873054504, + "corr_frobenius": 18.60379409790039, + "lag1_corr_mean_abs_diff": 0.24315960705280304, + "lag1_corr_frobenius": 19.07374382019043, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.35428014397621155, + "corr_frobenius": 12.232251167297363, + "lag1_corr_mean_abs_diff": 0.37317487597465515, + "lag1_corr_frobenius": 12.860593795776367 + }, + "P2": { + "corr_mean_abs_diff": 0.1587069034576416, + "corr_frobenius": 3.2476303577423096, + "lag1_corr_mean_abs_diff": 0.17968608438968658, + "lag1_corr_frobenius": 3.1598639488220215 + }, + "P3": { + "corr_mean_abs_diff": 0.29451829195022583, + "corr_frobenius": 2.0890676975250244, + "lag1_corr_mean_abs_diff": 0.3015052080154419, + "lag1_corr_frobenius": 2.1081626415252686 + }, + "P4": { + "corr_mean_abs_diff": 0.3379024267196655, + "corr_frobenius": 4.841367721557617, + "lag1_corr_mean_abs_diff": 0.37427571415901184, + "lag1_corr_frobenius": 5.122097492218018 + } + } + }, + "frequency": { + "avg_psd_l1": 0.019801901653409004, + "avg_psd_cosine": 0.420957660618818, + "avg_low_high_ratio_abs_diff": 118.50159391702495 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 308.0, + "mean_dwell": 1.2427184466019416, + "median_dwell": 1.0, + "mean_step": 0.8413416007896523, + "median_step": 0.7274894714355469 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 347.0, + "mean_dwell": 1.103448275862069, + "median_dwell": 1.0, + "mean_step": 41.3141472878254, + "median_step": 54.116028785705566 + }, + "P1_B2004": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 0.014068839402653235, + "median_step": 0.0017450004816055298 + }, + "P1_B3004": { + "num_changes": 370.0, + "mean_dwell": 1.0350404312668464, + "median_dwell": 1.0, + "mean_step": 14.77256890374261, + "median_step": 10.804779052734375 + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.8157603410765646, + "median_step": 0.5944099426269531 + }, + "P1_B3005": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 64.28072175926926, + "median_step": 41.228179931640625 + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22997.85714285714, + "mean_abs_diff_mean_dwell": 14417.281142645239, + "mean_abs_diff_median_dwell": 13988.57142857143, + "mean_abs_diff_mean_step": 19.03385039702909, + "mean_abs_diff_median_step": 16.840154950817425 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6822916666666666, + "change_rate": 0.7702349869451697, + "step_median": 68.81373023986816 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.09333292643229163, + "mean_abs_diff_change_rate": 0.309414869553025, + "mean_abs_diff_step_median": 68.78170993924141 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.015625, + "top1_mass": 0.40625, + "top3_mass": 0.7760416666666666, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8776041666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.059895833333333336, + "top3_mass": 0.0703125, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.18229166666666666, + "top3_mass": 0.3697916666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.8463541666666666, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4817708333333333, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.7864583333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3515625, + "top1_mass": 0.036458333333333336, + "top3_mass": 0.09635416666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5303484598795573, + "mean_abs_diff_top1_mass": 0.03112411499023438, + "mean_abs_diff_top3_mass": 0.05629857381184896, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": -0.10307999700307846, + "q50": 0.3064199984073639, + "q95": 0.5731499791145325, + "tail_ratio": 0.6513552715423715 + }, + "P2_SIT02": { + "q05": 779.8792114257812, + "q50": 785.1359252929688, + "q95": 789.8136596679688, + "tail_ratio": 0.889859043726633 + }, + "P1_FT03": { + "q05": 78.47207641601562, + "q50": 117.61963653564453, + "q95": 249.82940673828125, + "tail_ratio": 3.377216097213314 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 43.93644383301338, + "mean_abs_diff_q50": 62.870752066373825, + "mean_abs_diff_q95": 23.31601144870122, + "mean_abs_diff_tail_ratio": 8.098831336922286 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 64.74391126130756, + "median_step": 62.410614013671875 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 1312.3134692358592, + "median_step": 1284.6439208984375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.09039718638964156, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 680.877589673068, + "mean_abs_diff_median_step": 670.2603607177734 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 35.254512786865234, + "std": 31.650897979736328, + "lag1": -0.0927804708480835 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.003297113813459873, + "lag1": 0.06643600016832352 + }, + "P2_HILout": { + "mean": 711.4987182617188, + "std": 7.856828212738037, + "lag1": -0.013758387416601181 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.4565467834472656, + "mean_abs_diff_std": 0.2373291968833655, + "mean_abs_diff_lag1": 0.4579590285817782 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.8651301860809326, + "mae": 0.4152381420135498 + }, + "synthetic_only": { + "rmse": 1.4939647912979126, + "mae": 0.8916867971420288 + }, + "real_plus_synthetic": { + "rmse": 0.9160420298576355, + "mae": 0.4468558728694916 + }, + "rmse_ratio_synth_to_real": 1.726867025719702 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8564471243042671, + "auprc": 0.4132698022160485, + "threshold": 0.3127160668373108, + "f1": 0.21333333333333335, + "precision": 0.125, + "recall": 0.7272727272727273, + "best_f1": 0.5684210526315789, + "best_f1_threshold": 0.6349313259124756 + }, + "synthetic_only": { + "auroc": 0.803409090909091, + "auprc": 0.3912601416668211, + "threshold": 0.0337178111076355, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.5168539325842696, + "best_f1_threshold": 2.754981756210327 + }, + "real_plus_synthetic": { + "auroc": 0.8491419294990722, + "auprc": 0.414894592762339, + "threshold": 0.2921687662601471, + "f1": 0.17433414043583537, + "precision": 0.0975609756097561, + "recall": 0.8181818181818182, + "best_f1": 0.5684210526315789, + "best_f1_threshold": 0.6208351850509644 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_quantile__seed1337/eval.json b/example/results/ablations/runs/no_quantile__seed1337/eval.json new file mode 100644 index 0000000..7cb34d7 --- /dev/null +++ b/example/results/ablations/runs/no_quantile__seed1337/eval.json @@ -0,0 +1,974 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.10121, + "std": 0.0 + }, + "P1_B2016": { + "mean": 1.36826609375, + "std": 0.11514677714347482 + }, + "P1_B3004": { + "mean": 397.63785, + "std": 0.0 + }, + "P1_B3005": { + "mean": 1001.99799, + "std": 0.0 + }, + "P1_B4002": { + "mean": 33.6555, + "std": 0.0 + }, + "P1_B4005": { + "mean": 110.32928916666673, + "std": 21.209772564054123 + }, + "P1_B400B": { + "mean": 3416.8118279687505, + "std": 658.4866988945513 + }, + "P1_B4022": { + "mean": 37.07456895833334, + "std": 0.1625858665988879 + }, + "P1_FCV02Z": { + "mean": -32.49385838541669, + "std": 15.242345933983334 + }, + "P1_FCV03D": { + "mean": 37.49565908854168, + "std": 6.2405704684417636 + }, + "P1_FCV03Z": { + "mean": 38.25372846354165, + "std": 6.09190582166826 + }, + "P1_FT01": { + "mean": 48.651001015625, + "std": 39.649205778866055 + }, + "P1_FT01Z": { + "mean": 439.0570858333332, + "std": 121.73074514114414 + }, + "P1_FT02": { + "mean": 2395.205417161458, + "std": 494.87379435244907 + }, + "P1_FT02Z": { + "mean": 2395.205417161458, + "std": 494.87379435244907 + }, + "P1_FT03": { + "mean": 122.57399249999997, + "std": 47.90186702721992 + }, + "P1_FT03Z": { + "mean": 122.57399249999997, + "std": 47.90186702721992 + }, + "P1_LCV01D": { + "mean": 4.066330000000002, + "std": 1.7691601052594914 + }, + "P1_LCV01Z": { + "mean": 1.9114013281250017, + "std": 2.2158779634384835 + }, + "P1_LIT01": { + "mean": 372.67841315104175, + "std": 16.67141228151998 + }, + "P1_PCV01D": { + "mean": -31.283391666666652, + "std": 21.78768412939701 + }, + "P1_PCV01Z": { + "mean": -31.129499427083346, + "std": 22.094751901988932 + }, + "P1_PCV02Z": { + "mean": 11.994848307291663, + "std": 0.0041165894397556845 + }, + "P1_PIT01": { + "mean": 1.5574970833333341, + "std": 0.13360030473548123 + }, + "P1_PIT02": { + "mean": 0.2834835416666666, + "std": 0.17770432666324976 + }, + "P1_TIT01": { + "mean": 36.37579648437499, + "std": 0.23692429657147043 + }, + "P1_TIT02": { + "mean": 37.96692018229167, + "std": 0.7466679390743095 + }, + "P2_24Vdc": { + "mean": 28.029874661458326, + "std": 0.0032624608101602046 + }, + "P2_CO_rpm": { + "mean": 54098.85718749998, + "std": 21.127002258841436 + }, + "P2_HILout": { + "mean": 711.7379582812499, + "std": 10.349284391352574 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 784.5050000000002, + "std": 2.7899647657731954 + }, + "P2_SIT02": { + "mean": 784.938246744792, + "std": 3.030155698080626 + }, + "P2_VT01": { + "mean": 11.962031875000003, + "std": 0.030386675254603233 + }, + "P2_VXT02": { + "mean": -2.7612545052083326, + "std": 0.22704602290904505 + }, + "P2_VXT03": { + "mean": -0.608252239583333, + "std": 0.1882723210798349 + }, + "P2_VYT02": { + "mean": 4.281282291666663, + "std": 0.27228711825206514 + }, + "P2_VYT03": { + "mean": 6.417153229166667, + "std": 0.3109031766451265 + }, + "P3_FIT01": { + "mean": 1709.5997135416662, + "std": 2005.791684315084 + }, + "P3_LCP01D": { + "mean": 7303.585937499997, + "std": 2497.1332112203777 + }, + "P3_LCV01D": { + "mean": 5007.481770833336, + "std": 2555.242022981675 + }, + "P3_LIT01": { + "mean": 11495.241197916674, + "std": 1157.0713476064584 + }, + "P3_PIT01": { + "mean": 48.490026041666646, + "std": 210.54386189328645 + }, + "P4_HT_FD": { + "mean": 0.0023197656249999998, + "std": 0.002926024615303769 + }, + "P4_HT_LD": { + "mean": -0.004517604166666671, + "std": 0.011271361314850317 + }, + "P4_HT_PO": { + "mean": 33.0370705989583, + "std": 12.766562744589022 + }, + "P4_LD": { + "mean": 300.8689323437499, + "std": 57.53204552624241 + }, + "P4_ST_FD": { + "mean": -0.00035153645833333315, + "std": 0.0019591724377820787 + }, + "P4_ST_GOV": { + "mean": 16993.95177083333, + "std": 1770.0428398487213 + }, + "P4_ST_LD": { + "mean": 290.75086945312495, + "std": 44.47131476093578 + }, + "P4_ST_PO": { + "mean": 319.0671400781251, + "std": 32.90894195377767 + }, + "P4_ST_PT01": { + "mean": 10111.798255208329, + "std": 36.35350842428432 + }, + "P4_ST_TT01": { + "mean": 27624.066406250007, + "std": 12.768416591222536 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.01471913179999973, + "std_abs_err": 0.024492489898690458 + }, + "P1_B2016": { + "mean_abs_err": 0.007895362250001092, + "std_abs_err": 0.01434594850412263 + }, + "P1_B3004": { + "mean_abs_err": 1.4516903093982023, + "std_abs_err": 10.16264800653289 + }, + "P1_B3005": { + "mean_abs_err": 35.374394413793084, + "std_abs_err": 70.85697659109 + }, + "P1_B4002": { + "mean_abs_err": 1.0906270592000098, + "std_abs_err": 0.7578213113008355 + }, + "P1_B4005": { + "mean_abs_err": 44.34738159426625, + "std_abs_err": 20.590880585863847 + }, + "P1_B400B": { + "mean_abs_err": 1491.7726709441572, + "std_abs_err": 518.1578558503119 + }, + "P1_B4022": { + "mean_abs_err": 0.7854882903333262, + "std_abs_err": 0.6595256400498211 + }, + "P1_FCV02Z": { + "mean_abs_err": 54.23811950381672, + "std_abs_err": 23.876086043658436 + }, + "P1_FCV03D": { + "mean_abs_err": 19.865573652858764, + "std_abs_err": 1.6489369792848612 + }, + "P1_FCV03Z": { + "mean_abs_err": 19.79711673285837, + "std_abs_err": 1.9541630842774564 + }, + "P1_FT01": { + "mean_abs_err": 135.5351501075723, + "std_abs_err": 8.848035460037494 + }, + "P1_FT01Z": { + "mean_abs_err": 412.8210892372633, + "std_abs_err": 30.45205859781234 + }, + "P1_FT02": { + "mean_abs_err": 1139.3481998070513, + "std_abs_err": 384.8425333810003 + }, + "P1_FT02Z": { + "mean_abs_err": 470.1843416420468, + "std_abs_err": 681.7961587780626 + }, + "P1_FT03": { + "mean_abs_err": 146.79886635780576, + "std_abs_err": 9.721708607570513 + }, + "P1_FT03Z": { + "mean_abs_err": 914.792179730601, + "std_abs_err": 22.829140717144355 + }, + "P1_LCV01D": { + "mean_abs_err": 7.162519048599961, + "std_abs_err": 1.5664054362962683 + }, + "P1_LCV01Z": { + "mean_abs_err": 9.080208853475014, + "std_abs_err": 1.1704542703350613 + }, + "P1_LIT01": { + "mean_abs_err": 24.206117959957623, + "std_abs_err": 6.092697521415859 + }, + "P1_PCV01D": { + "mean_abs_err": 85.08440785086651, + "std_abs_err": 2.1720046932581596 + }, + "P1_PCV01Z": { + "mean_abs_err": 85.77613971428295, + "std_abs_err": 2.315997434686846 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.022925235508408193, + "std_abs_err": 0.000688208453404315 + }, + "P1_PIT01": { + "mean_abs_err": 0.1882111345333266, + "std_abs_err": 0.05593880933016994 + }, + "P1_PIT02": { + "mean_abs_err": 0.16110717093333565, + "std_abs_err": 0.2705279914932733 + }, + "P1_TIT01": { + "mean_abs_err": 0.7332383443751098, + "std_abs_err": 0.36174355618141096 + }, + "P1_TIT02": { + "mean_abs_err": 1.5188419516914422, + "std_abs_err": 0.44256618137779535 + }, + "P2_24Vdc": { + "mean_abs_err": 0.0018727600584256265, + "std_abs_err": 5.361830606238861e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 6.78716249998979, + "std_abs_err": 0.551524437334102 + }, + "P2_HILout": { + "mean_abs_err": 0.32090846134224194, + "std_abs_err": 2.1704310114439664 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 5.728014999998891, + "std_abs_err": 1.1045710098940607 + }, + "P2_SIT02": { + "mean_abs_err": 6.1603531975938495, + "std_abs_err": 0.8523213807773136 + }, + "P2_VT01": { + "mean_abs_err": 0.047082426799958554, + "std_abs_err": 0.037743233912099244 + }, + "P2_VXT02": { + "mean_abs_err": 0.7655326887916849, + "std_abs_err": 0.20399554826664298 + }, + "P2_VXT03": { + "mean_abs_err": 0.9438382526166584, + "std_abs_err": 0.08067019850156285 + }, + "P2_VYT02": { + "mean_abs_err": 0.4851695540666614, + "std_abs_err": 0.18880367006869347 + }, + "P2_VYT03": { + "mean_abs_err": 0.29546153216664894, + "std_abs_err": 0.004938882794371768 + }, + "P3_FIT01": { + "mean_abs_err": 541.3468335416649, + "std_abs_err": 218.49291500089726 + }, + "P3_LCP01D": { + "mean_abs_err": 2628.1206975000086, + "std_abs_err": 2648.276214960895 + }, + "P3_LCV01D": { + "mean_abs_err": 2437.726949166681, + "std_abs_err": 4230.360758783421 + }, + "P3_LIT01": { + "mean_abs_err": 2233.7411170831783, + "std_abs_err": 2903.8440942662864 + }, + "P3_PIT01": { + "mean_abs_err": 620.4822089583337, + "std_abs_err": 957.5632645491162 + }, + "P4_HT_FD": { + "mean_abs_err": 0.0024198914250000004, + "std_abs_err": 0.0008934422346861771 + }, + "P4_HT_LD": { + "mean_abs_err": 35.4239676051662, + "std_abs_err": 33.20108980793838 + }, + "P4_HT_PO": { + "mean_abs_err": 2.371499392241901, + "std_abs_err": 18.421263169926135 + }, + "P4_LD": { + "mean_abs_err": 64.51444223664868, + "std_abs_err": 2.2045710628032325 + }, + "P4_ST_FD": { + "mean_abs_err": 0.0003450158583333332, + "std_abs_err": 0.00031629372503882915 + }, + "P4_ST_GOV": { + "mean_abs_err": 807.8611741666318, + "std_abs_err": 29.443094035900003 + }, + "P4_ST_LD": { + "mean_abs_err": 39.081722728874695, + "std_abs_err": 8.60498187193501 + }, + "P4_ST_PO": { + "mean_abs_err": 11.040806071671625, + "std_abs_err": 0.5339292185209743 + }, + "P4_ST_PT01": { + "mean_abs_err": 64.11865020820187, + "std_abs_err": 13.893545606138073 + }, + "P4_ST_TT01": { + "mean_abs_err": 17.20633624985203, + "std_abs_err": 11.977522758998951 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.1894, + "P1_B2016": 0.30232083333333337, + "P1_B3004": 0.18945, + "P1_B3005": 0.18945, + "P1_B4002": 0.1895, + "P1_B4005": 0.6901041666666667, + "P1_B400B": 0.9479166666666666, + "P1_B4022": 0.44731666666666664, + "P1_FCV02Z": 0.9557291666666666, + "P1_FCV03D": 0.9259333333333334, + "P1_FCV03Z": 0.9248333333333334, + "P1_FT01": 0.9074166666666666, + "P1_FT01Z": 0.9041666666666667, + "P1_FT02": 0.9478708333333333, + "P1_FT02Z": 0.6634875, + "P1_FT03": 0.9375, + "P1_FT03Z": 1.0, + "P1_LCV01D": 0.9166666666666666, + "P1_LCV01Z": 0.9200833333333334, + "P1_LIT01": 0.7643666666666666, + "P1_PCV01D": 0.9479166666666666, + "P1_PCV01Z": 0.9479166666666666, + "P1_PCV02Z": 0.96405, + "P1_PIT01": 0.6181625, + "P1_PIT02": 0.5325, + "P1_TIT01": 0.5590208333333334, + "P1_TIT02": 0.6881625, + "P2_24Vdc": 0.29181250000000003, + "P2_CO_rpm": 0.15797499999999998, + "P2_HILout": 0.08944583333333335, + "P2_MSD": 0.0, + "P2_SIT01": 0.6598333333333334, + "P2_SIT02": 0.6629958333333333, + "P2_VT01": 0.4668916666666666, + "P2_VXT02": 0.9322583333333334, + "P2_VXT03": 0.9784166666666666, + "P2_VYT02": 0.5474958333333334, + "P2_VYT03": 0.5795333333333332, + "P3_FIT01": 0.3636125, + "P3_LCP01D": 0.57795, + "P3_LCV01D": 0.49150000000000005, + "P3_LIT01": 0.44431666666666664, + "P3_PIT01": 0.4088541666666667, + "P4_HT_FD": 0.5907875, + "P4_HT_LD": 0.6954, + "P4_HT_PO": 0.4130541666666666, + "P4_LD": 0.4144958333333334, + "P4_ST_FD": 0.168875, + "P4_ST_GOV": 0.4306625, + "P4_ST_LD": 0.51475, + "P4_ST_PO": 0.38827083333333334, + "P4_ST_PT01": 0.7711666666666667, + "P4_ST_TT01": 0.515625 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.0707, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B2016": { + "q05_diff": 0.11887999999999987, + "q25_diff": 0.012909999999999977, + "q50_diff": 0.0697000000000001, + "q75_diff": 0.026550000000000074, + "q95_diff": 0.04960999999999993 + }, + "P1_B3004": { + "q05_diff": 4.475590000000011, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B3005": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 113.12176000000011 + }, + "P1_B4002": { + "q05_diff": 1.6555000000000035, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B4005": { + "q05_diff": 91.23792, + "q25_diff": 1.1653400000000005, + "q50_diff": 3.422150000000002, + "q75_diff": 11.597669999999994, + "q95_diff": 62.72931 + }, + "P1_B400B": { + "q05_diff": 2718.85966, + "q25_diff": 200.6357399999997, + "q50_diff": 399.82324000000017, + "q75_diff": 690.95606, + "q95_diff": 2133.9103999999998 + }, + "P1_B4022": { + "q05_diff": 1.3686399999999992, + "q25_diff": 0.006489999999999441, + "q50_diff": 0.28572000000000486, + "q75_diff": 0.36582999999999544, + "q95_diff": 0.4306800000000024 + }, + "P1_FCV02Z": { + "q05_diff": 57.74044, + "q25_diff": 40.57549, + "q50_diff": 27.177329999999998, + "q75_diff": 23.56168, + "q95_diff": 105.6484 + }, + "P1_FCV03D": { + "q05_diff": 19.01674, + "q25_diff": 18.620870000000004, + "q50_diff": 15.529209999999999, + "q75_diff": 12.119680000000002, + "q95_diff": 13.128130000000006 + }, + "P1_FCV03Z": { + "q05_diff": 18.576789999999995, + "q25_diff": 18.062510000000003, + "q50_diff": 15.1873, + "q75_diff": 12.377589999999998, + "q95_diff": 13.75231999999999 + }, + "P1_FT01": { + "q05_diff": 120.72912000000001, + "q25_diff": 135.22494, + "q50_diff": 139.04176, + "q75_diff": 136.48351, + "q95_diff": 55.99644999999998 + }, + "P1_FT01Z": { + "q05_diff": 387.58875, + "q25_diff": 420.63922, + "q50_diff": 434.99024, + "q75_diff": 448.03643999999997, + "q95_diff": 136.61852999999996 + }, + "P1_FT02": { + "q05_diff": 1991.54888, + "q25_diff": 139.78015000000005, + "q50_diff": 286.57434999999987, + "q75_diff": 414.56177, + "q95_diff": 1577.1709 + }, + "P1_FT02Z": { + "q05_diff": 1964.1081900000001, + "q25_diff": 722.1606400000001, + "q50_diff": 573.0, + "q75_diff": 443.30029000000013, + "q95_diff": 721.47388 + }, + "P1_FT03": { + "q05_diff": 166.6325, + "q25_diff": 161.89179, + "q50_diff": 130.77871, + "q75_diff": 112.34787, + "q95_diff": 67.71792999999997 + }, + "P1_FT03Z": { + "q05_diff": 919.36636, + "q25_diff": 916.0905, + "q50_diff": 886.19805, + "q75_diff": 869.72023, + "q95_diff": 869.82565 + }, + "P1_LCV01D": { + "q05_diff": 5.07595, + "q25_diff": 4.59192, + "q50_diff": 4.89555, + "q75_diff": 6.74542, + "q95_diff": 7.39312 + }, + "P1_LCV01Z": { + "q05_diff": 6.77731, + "q25_diff": 6.70737, + "q50_diff": 6.95468, + "q75_diff": 8.906450000000001, + "q95_diff": 7.823410000000001 + }, + "P1_LIT01": { + "q05_diff": 41.02197000000001, + "q25_diff": 39.912379999999985, + "q50_diff": 25.052059999999983, + "q75_diff": 11.556579999999997, + "q95_diff": 6.819880000000012 + }, + "P1_PCV01D": { + "q05_diff": 80.22999999999999, + "q25_diff": 84.68, + "q50_diff": 86.55514, + "q75_diff": 72.77674, + "q95_diff": 27.149800000000003 + }, + "P1_PCV01Z": { + "q05_diff": 80.70954, + "q25_diff": 85.33295000000001, + "q50_diff": 88.41523000000001, + "q75_diff": 73.40637, + "q95_diff": 27.29011 + }, + "P1_PCV02Z": { + "q05_diff": 0.016439999999999344, + "q25_diff": 0.016439999999999344, + "q50_diff": 0.024070000000000036, + "q75_diff": 0.024070000000000036, + "q95_diff": 0.012980000000000658 + }, + "P1_PIT01": { + "q05_diff": 0.12588, + "q25_diff": 0.09983000000000009, + "q50_diff": 0.14529999999999998, + "q75_diff": 0.28562, + "q95_diff": 0.27505 + }, + "P1_PIT02": { + "q05_diff": 0.3077, + "q25_diff": 0.09689999999999999, + "q50_diff": 0.029230000000000034, + "q75_diff": 0.09885, + "q95_diff": 0.29157999999999995 + }, + "P1_TIT01": { + "q05_diff": 1.1499400000000009, + "q25_diff": 1.0675299999999979, + "q50_diff": 0.7149500000000018, + "q75_diff": 0.24499999999999744, + "q95_diff": 0.0903199999999984 + }, + "P1_TIT02": { + "q05_diff": 1.5033499999999975, + "q25_diff": 1.9118800000000036, + "q50_diff": 1.714420000000004, + "q75_diff": 1.4456300000000013, + "q95_diff": 0.03227000000000402 + }, + "P2_24Vdc": { + "q05_diff": 0.0015299999999989211, + "q25_diff": 0.0020500000000005514, + "q50_diff": 0.0015199999999992997, + "q75_diff": 0.0020199999999981344, + "q95_diff": 0.002449999999999619 + }, + "P2_CO_rpm": { + "q05_diff": 11.330000000001746, + "q25_diff": 8.69999999999709, + "q50_diff": 5.690000000002328, + "q75_diff": 7.30000000000291, + "q95_diff": 4.44999999999709 + }, + "P2_HILout": { + "q05_diff": 6.2588499999999385, + "q25_diff": 1.794369999999958, + "q50_diff": 1.1411699999999882, + "q75_diff": 0.18639999999993506, + "q95_diff": 1.2944299999999203 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 7.360000000000014, + "q25_diff": 5.57000000000005, + "q50_diff": 5.840000000000032, + "q75_diff": 5.5, + "q95_diff": 3.3899999999999864 + }, + "P2_SIT02": { + "q05_diff": 7.619019999999978, + "q25_diff": 6.217650000000049, + "q50_diff": 6.172910000000002, + "q75_diff": 5.812379999999962, + "q95_diff": 5.001769999999965 + }, + "P2_VT01": { + "q05_diff": 0.08244000000000007, + "q25_diff": 0.10481000000000051, + "q50_diff": 0.05597999999999992, + "q75_diff": 0.0054800000000003735, + "q95_diff": 0.0011799999999997368 + }, + "P2_VXT02": { + "q05_diff": 0.9418600000000001, + "q25_diff": 1.1576499999999998, + "q50_diff": 0.8090000000000002, + "q75_diff": 0.51065, + "q95_diff": 0.5369000000000002 + }, + "P2_VXT03": { + "q05_diff": 1.15339, + "q25_diff": 1.1620499999999998, + "q50_diff": 0.9971699999999999, + "q75_diff": 0.83748, + "q95_diff": 1.09912 + }, + "P2_VYT02": { + "q05_diff": 0.41571999999999987, + "q25_diff": 0.9217900000000001, + "q50_diff": 0.6209599999999997, + "q75_diff": 0.2496299999999998, + "q95_diff": 0.2185100000000002 + }, + "P2_VYT03": { + "q05_diff": 0.057640000000000136, + "q25_diff": 0.4686199999999996, + "q50_diff": 0.46452000000000027, + "q75_diff": 0.2946100000000005, + "q95_diff": 0.28494999999999937 + }, + "P3_FIT01": { + "q05_diff": 928.66, + "q25_diff": 256.27, + "q50_diff": 1503.74, + "q75_diff": 754.9899999999998, + "q95_diff": 193.55000000000018 + }, + "P3_LCP01D": { + "q05_diff": 3916.0, + "q25_diff": 5120.0, + "q50_diff": 5198.0, + "q75_diff": 102.0, + "q95_diff": 2435.0 + }, + "P3_LCV01D": { + "q05_diff": 1945.0, + "q25_diff": 2748.0, + "q50_diff": 4739.0, + "q75_diff": 7291.0, + "q95_diff": 6816.0 + }, + "P3_LIT01": { + "q05_diff": 3192.3999999999996, + "q25_diff": 757.8799999999992, + "q50_diff": 1750.5100000000002, + "q75_diff": 3869.7700000000004, + "q95_diff": 5256.780000000001 + }, + "P3_PIT01": { + "q05_diff": 234.13, + "q25_diff": 124.66999999999999, + "q50_diff": 66.61, + "q75_diff": 702.55, + "q95_diff": 3113.87 + }, + "P4_HT_FD": { + "q05_diff": 0.0013199999999999998, + "q25_diff": 3.9999999999999996e-05, + "q50_diff": 0.0037099999999999998, + "q75_diff": 0.0044800000000000005, + "q95_diff": 0.0047599999999999995 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 54.74537, + "q75_diff": 68.58362000000001, + "q95_diff": 76.30932 + }, + "P4_HT_PO": { + "q05_diff": 11.97899, + "q25_diff": 22.97427, + "q50_diff": 13.187750000000005, + "q75_diff": 24.267110000000002, + "q95_diff": 20.974529999999994 + }, + "P4_LD": { + "q05_diff": 40.30498, + "q25_diff": 80.16645, + "q50_diff": 73.74402000000003, + "q75_diff": 50.643100000000004, + "q95_diff": 78.09792999999996 + }, + "P4_ST_FD": { + "q05_diff": 0.0007900000000000003, + "q25_diff": 0.0006099999999999999, + "q50_diff": 0.00058, + "q75_diff": 0.0001600000000000001, + "q95_diff": 0.0006099999999999999 + }, + "P4_ST_GOV": { + "q05_diff": 653.1599999999999, + "q25_diff": 1090.1299999999992, + "q50_diff": 1825.2299999999996, + "q75_diff": 659.5400000000009, + "q95_diff": 273.40999999999985 + }, + "P4_ST_LD": { + "q05_diff": 25.522340000000014, + "q25_diff": 54.95673000000002, + "q50_diff": 52.53005999999999, + "q75_diff": 30.317200000000014, + "q95_diff": 3.5045499999999947 + }, + "P4_ST_PO": { + "q05_diff": 16.012389999999982, + "q25_diff": 16.468050000000005, + "q50_diff": 30.65924000000001, + "q75_diff": 9.563509999999951, + "q95_diff": 11.301789999999983 + }, + "P4_ST_PT01": { + "q05_diff": 24.659999999999854, + "q25_diff": 55.17000000000007, + "q50_diff": 55.30999999999949, + "q75_diff": 101.72999999999956, + "q95_diff": 76.97999999999956 + }, + "P4_ST_TT01": { + "q05_diff": 50.419999999998254, + "q25_diff": 25.590000000000146, + "q50_diff": 1.5400000000008731, + "q75_diff": 7.9099999999998545, + "q95_diff": 12.450000000000728 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 0.9999898105231887, + "P1_B2016": 0.005256433366776614, + "P1_B3004": 0.9999898125949512, + "P1_B3005": 0.9999898125942444, + "P1_B4002": 0.9999898146591784, + "P1_B4005": 0.030658498303901216, + "P1_B400B": 0.028092634631170177, + "P1_B4022": 0.022255348150370113, + "P1_FCV02Z": 0.1075232422240463, + "P1_FCV03D": 0.07023839671002341, + "P1_FCV03Z": 0.07319651353736367, + "P1_FT01": 0.04946603703802366, + "P1_FT01Z": 0.05045622563091379, + "P1_FT02": 0.02664690203439979, + "P1_FT02Z": 0.026701880644311338, + "P1_FT03": 0.06818052959716525, + "P1_FT03Z": 0.06804347255255838, + "P1_LCV01D": 0.06943161529628272, + "P1_LCV01Z": 0.07198514280503232, + "P1_LIT01": 0.032560279359611255, + "P1_PCV01D": 0.0851891166055243, + "P1_PCV01Z": 0.08398043639362618, + "P1_PCV02Z": 0.30996734709091944, + "P1_PIT01": 0.0021008931037195255, + "P1_PIT02": 0.013277356227836434, + "P1_TIT01": 0.013176470128318685, + "P1_TIT02": 0.0066781343327798615, + "P2_24Vdc": 0.08554521051562067, + "P2_CO_rpm": 0.2763124243951074, + "P2_HILout": 0.18488250640096165, + "P2_MSD": 0.0, + "P2_SIT01": 0.3958485218427761, + "P2_SIT02": 0.5077622761106568, + "P2_VT01": 0.09799551246438576, + "P2_VXT02": 0.08617199848952684, + "P2_VXT03": 0.020537043704080604, + "P2_VYT02": 0.004882186781516085, + "P2_VYT03": 0.11977875042928843, + "P3_FIT01": 0.015510481846798818, + "P3_LCP01D": 0.028799148940010566, + "P3_LCV01D": 0.02734570443173201, + "P3_LIT01": 0.06146827249625708, + "P3_PIT01": 0.021691058833552712, + "P4_HT_FD": 0.48478468436597033, + "P4_HT_LD": 0.24400140093685518, + "P4_HT_PO": 0.020554571451316894, + "P4_LD": 0.025261119593570136, + "P4_ST_FD": 0.17359710056794989, + "P4_ST_GOV": 0.009656930397496155, + "P4_ST_LD": 0.018143802080119564, + "P4_ST_PO": 0.008112295501195299, + "P4_ST_PT01": 0.003276191275868312, + "P4_ST_TT01": 0.015004036696998568 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.09144631115705412, + "P1_FCV02D": 0.07101077239149962, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + }, + "avg_ks": 0.5986268081761006, + "avg_jsd": 0.010030341725636, + "avg_lag1_diff": 0.15569708276765754 +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_quantile__seed1337/eval_post.json b/example/results/ablations/runs/no_quantile__seed1337/eval_post.json new file mode 100644 index 0000000..caf7a4e --- /dev/null +++ b/example/results/ablations/runs/no_quantile__seed1337/eval_post.json @@ -0,0 +1,971 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09355486979166669, + "std": 0.01799194293084728 + }, + "P1_B2016": { + "mean": 1.36826609375, + "std": 0.11514677714347482 + }, + "P1_B3004": { + "mean": 403.2905014583335, + "std": 14.217563527951576 + }, + "P1_B3005": { + "mean": 1059.5253898958326, + "std": 58.42587703751758 + }, + "P1_B4002": { + "mean": 32.37490617187499, + "std": 0.6519533213854061 + }, + "P1_B4005": { + "mean": 64.99571291666655, + "std": 42.03467331899641 + }, + "P1_B400B": { + "mean": 3416.8118279687505, + "std": 658.4866988945513 + }, + "P1_B4022": { + "mean": 36.08671476562498, + "std": 0.7147782383668128 + }, + "P1_FCV02Z": { + "mean": 21.954832630208315, + "std": 38.87335152465215 + }, + "P1_FCV03D": { + "mean": 58.14541601562505, + "std": 6.267267768996455 + }, + "P1_FCV03Z": { + "mean": 58.21398343750003, + "std": 6.423211358701469 + }, + "P1_FT01": { + "mean": 48.651001015625, + "std": 39.649205778866055 + }, + "P1_FT01Z": { + "mean": 439.0570858333332, + "std": 121.73074514114414 + }, + "P1_FT02": { + "mean": 2395.205417161458, + "std": 494.87379435244907 + }, + "P1_FT02Z": { + "mean": 1843.7261292187509, + "std": 1190.4241853713818 + }, + "P1_FT03": { + "mean": 122.57399249999997, + "std": 47.90186702721992 + }, + "P1_FT03Z": { + "mean": 1058.2564329947922, + "std": 59.50957255113764 + }, + "P1_LCV01D": { + "mean": 11.699055963541667, + "std": 3.0155193220433567 + }, + "P1_LCV01Z": { + "mean": 11.627510442708333, + "std": 3.0251072332459787 + }, + "P1_LIT01": { + "mean": 372.67841315104175, + "std": 16.67141228151998 + }, + "P1_PCV01D": { + "mean": 65.64593611979173, + "std": 23.56629129628467 + }, + "P1_PCV01Z": { + "mean": 65.565628671875, + "std": 23.790956943468952 + }, + "P1_PCV02Z": { + "mean": 11.867279765625007, + "std": 0.10811550492611105 + }, + "P1_PIT01": { + "mean": 1.5574970833333341, + "std": 0.13360030473548123 + }, + "P1_PIT02": { + "mean": 0.2834835416666666, + "std": 0.17770432666324976 + }, + "P1_TIT01": { + "mean": 36.37579648437499, + "std": 0.23692429657147043 + }, + "P1_TIT02": { + "mean": 37.96692018229167, + "std": 0.7466679390743095 + }, + "P2_24Vdc": { + "mean": 28.02860809895833, + "std": 0.0032970984602375326 + }, + "P2_CO_rpm": { + "mean": 54098.85718749998, + "std": 21.127002258841436 + }, + "P2_HILout": { + "mean": 711.4986896354167, + "std": 7.856828438379631 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 784.5050000000002, + "std": 2.7899647657731954 + }, + "P2_SIT02": { + "mean": 784.938246744792, + "std": 3.030155698080626 + }, + "P2_VT01": { + "mean": 11.962031875000003, + "std": 0.030386675254603233 + }, + "P2_VXT02": { + "mean": -2.7612545052083326, + "std": 0.22704602290904505 + }, + "P2_VXT03": { + "mean": -0.608252239583333, + "std": 0.1882723210798349 + }, + "P2_VYT02": { + "mean": 4.281282291666663, + "std": 0.27228711825206514 + }, + "P2_VYT03": { + "mean": 6.417153229166667, + "std": 0.3109031766451265 + }, + "P3_FIT01": { + "mean": 1709.5997135416662, + "std": 2005.791684315084 + }, + "P3_LCP01D": { + "mean": 7303.585937499997, + "std": 2497.1332112203777 + }, + "P3_LCV01D": { + "mean": 5007.481770833336, + "std": 2555.242022981675 + }, + "P3_LIT01": { + "mean": 11495.241197916674, + "std": 1157.0713476064584 + }, + "P3_PIT01": { + "mean": 48.490026041666646, + "std": 210.54386189328645 + }, + "P4_HT_FD": { + "mean": 0.0023197656249999998, + "std": 0.002926024615303769 + }, + "P4_HT_LD": { + "mean": 34.231568880208286, + "std": 33.60361854009684 + }, + "P4_HT_PO": { + "mean": 35.25451283854166, + "std": 31.650898099038645 + }, + "P4_LD": { + "mean": 300.8689323437499, + "std": 57.53204552624241 + }, + "P4_ST_FD": { + "mean": -0.00035153645833333315, + "std": 0.0019591724377820787 + }, + "P4_ST_GOV": { + "mean": 16993.95177083333, + "std": 1770.0428398487213 + }, + "P4_ST_LD": { + "mean": 290.75086945312495, + "std": 44.47131476093578 + }, + "P4_ST_PO": { + "mean": 319.0671400781251, + "std": 32.90894195377767 + }, + "P4_ST_PT01": { + "mean": 10111.798255208329, + "std": 36.35350842428432 + }, + "P4_ST_TT01": { + "mean": 27624.066406250007, + "std": 12.768416591222536 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.0070640015916664245, + "std_abs_err": 0.0065005469678431765 + }, + "P1_B2016": { + "mean_abs_err": 0.007895362250001092, + "std_abs_err": 0.01434594850412263 + }, + "P1_B3004": { + "mean_abs_err": 7.1043417677316825, + "std_abs_err": 4.054915521418685 + }, + "P1_B3005": { + "mean_abs_err": 22.15300548203959, + "std_abs_err": 12.431099553572423 + }, + "P1_B4002": { + "mean_abs_err": 0.18996676892500375, + "std_abs_err": 0.10586798991542945 + }, + "P1_B4005": { + "mean_abs_err": 0.9861946557339252, + "std_abs_err": 0.23402016907844114 + }, + "P1_B400B": { + "mean_abs_err": 1491.7726709441572, + "std_abs_err": 518.1578558503119 + }, + "P1_B4022": { + "mean_abs_err": 0.20236590237502838, + "std_abs_err": 0.10733326828189615 + }, + "P1_FCV02Z": { + "mean_abs_err": 0.21057151180827915, + "std_abs_err": 0.2450804529896189 + }, + "P1_FCV03D": { + "mean_abs_err": 0.7841832742246098, + "std_abs_err": 1.6222396787301694 + }, + "P1_FCV03Z": { + "mean_abs_err": 0.16313824110001462, + "std_abs_err": 1.622857547244248 + }, + "P1_FT01": { + "mean_abs_err": 135.5351501075723, + "std_abs_err": 8.848035460037494 + }, + "P1_FT01Z": { + "mean_abs_err": 412.8210892372633, + "std_abs_err": 30.45205859781234 + }, + "P1_FT02": { + "mean_abs_err": 1139.3481998070513, + "std_abs_err": 384.8425333810003 + }, + "P1_FT02Z": { + "mean_abs_err": 81.29494630066051, + "std_abs_err": 13.754232240870124 + }, + "P1_FT03": { + "mean_abs_err": 146.79886635780576, + "std_abs_err": 9.721708607570513 + }, + "P1_FT03Z": { + "mean_abs_err": 20.890260764191225, + "std_abs_err": 11.22143519322664 + }, + "P1_LCV01D": { + "mean_abs_err": 0.47020691494170386, + "std_abs_err": 0.320046219512403 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.6359002611083167, + "std_abs_err": 0.3612250005275661 + }, + "P1_LIT01": { + "mean_abs_err": 24.206117959957623, + "std_abs_err": 6.092697521415859 + }, + "P1_PCV01D": { + "mean_abs_err": 11.844919935591868, + "std_abs_err": 3.9506118601458198 + }, + "P1_PCV01Z": { + "mean_abs_err": 10.918988384675409, + "std_abs_err": 4.012202476166866 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.15049377717506474, + "std_abs_err": 0.10331070703295105 + }, + "P1_PIT01": { + "mean_abs_err": 0.1882111345333266, + "std_abs_err": 0.05593880933016994 + }, + "P1_PIT02": { + "mean_abs_err": 0.16110717093333565, + "std_abs_err": 0.2705279914932733 + }, + "P1_TIT01": { + "mean_abs_err": 0.7332383443751098, + "std_abs_err": 0.36174355618141096 + }, + "P1_TIT02": { + "mean_abs_err": 1.5188419516914422, + "std_abs_err": 0.44256618137779535 + }, + "P2_24Vdc": { + "mean_abs_err": 0.0006061975584294998, + "std_abs_err": 8.825595613971664e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 6.78716249998979, + "std_abs_err": 0.551524437334102 + }, + "P2_HILout": { + "mean_abs_err": 0.5601771071754911, + "std_abs_err": 0.3220249415289764 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 5.728014999998891, + "std_abs_err": 1.1045710098940607 + }, + "P2_SIT02": { + "mean_abs_err": 6.1603531975938495, + "std_abs_err": 0.8523213807773136 + }, + "P2_VT01": { + "mean_abs_err": 0.047082426799958554, + "std_abs_err": 0.037743233912099244 + }, + "P2_VXT02": { + "mean_abs_err": 0.7655326887916849, + "std_abs_err": 0.20399554826664298 + }, + "P2_VXT03": { + "mean_abs_err": 0.9438382526166584, + "std_abs_err": 0.08067019850156285 + }, + "P2_VYT02": { + "mean_abs_err": 0.4851695540666614, + "std_abs_err": 0.18880367006869347 + }, + "P2_VYT03": { + "mean_abs_err": 0.29546153216664894, + "std_abs_err": 0.004938882794371768 + }, + "P3_FIT01": { + "mean_abs_err": 541.3468335416649, + "std_abs_err": 218.49291500089726 + }, + "P3_LCP01D": { + "mean_abs_err": 2628.1206975000086, + "std_abs_err": 2648.276214960895 + }, + "P3_LCV01D": { + "mean_abs_err": 2437.726949166681, + "std_abs_err": 4230.360758783421 + }, + "P3_LIT01": { + "mean_abs_err": 2233.7411170831783, + "std_abs_err": 2903.8440942662864 + }, + "P3_PIT01": { + "mean_abs_err": 620.4822089583337, + "std_abs_err": 957.5632645491162 + }, + "P4_HT_FD": { + "mean_abs_err": 0.0024198914250000004, + "std_abs_err": 0.0008934422346861771 + }, + "P4_HT_LD": { + "mean_abs_err": 1.1878811207912463, + "std_abs_err": 0.39125737084361134 + }, + "P4_HT_PO": { + "mean_abs_err": 0.15405715265854525, + "std_abs_err": 0.46307218452348664 + }, + "P4_LD": { + "mean_abs_err": 64.51444223664868, + "std_abs_err": 2.2045710628032325 + }, + "P4_ST_FD": { + "mean_abs_err": 0.0003450158583333332, + "std_abs_err": 0.00031629372503882915 + }, + "P4_ST_GOV": { + "mean_abs_err": 807.8611741666318, + "std_abs_err": 29.443094035900003 + }, + "P4_ST_LD": { + "mean_abs_err": 39.081722728874695, + "std_abs_err": 8.60498187193501 + }, + "P4_ST_PO": { + "mean_abs_err": 11.040806071671625, + "std_abs_err": 0.5339292185209743 + }, + "P4_ST_PT01": { + "mean_abs_err": 64.11865020820187, + "std_abs_err": 13.893545606138073 + }, + "P4_ST_TT01": { + "mean_abs_err": 17.20633624985203, + "std_abs_err": 11.977522758998951 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.6595583333333334, + "P1_B2016": 0.30232083333333337, + "P1_B3004": 0.671875, + "P1_B3005": 0.5293, + "P1_B4002": 0.7923208333333334, + "P1_B4005": 0.3062625, + "P1_B400B": 0.9479166666666666, + "P1_B4022": 0.5756333333333334, + "P1_FCV02Z": 0.7746, + "P1_FCV03D": 0.45162500000000005, + "P1_FCV03Z": 0.40985833333333327, + "P1_FT01": 0.9074166666666666, + "P1_FT01Z": 0.9041666666666667, + "P1_FT02": 0.9478708333333333, + "P1_FT02Z": 0.4029958333333334, + "P1_FT03": 0.9375, + "P1_FT03Z": 0.46950833333333336, + "P1_LCV01D": 0.3859916666666667, + "P1_LCV01Z": 0.397625, + "P1_LIT01": 0.7643666666666666, + "P1_PCV01D": 0.5304583333333333, + "P1_PCV01Z": 0.5184416666666667, + "P1_PCV02Z": 0.6588541666666666, + "P1_PIT01": 0.6181625, + "P1_PIT02": 0.5325, + "P1_TIT01": 0.5590208333333334, + "P1_TIT02": 0.6881625, + "P2_24Vdc": 0.13548333333333334, + "P2_CO_rpm": 0.15797499999999998, + "P2_HILout": 0.08242499999999997, + "P2_MSD": 0.0, + "P2_SIT01": 0.6598333333333334, + "P2_SIT02": 0.6629958333333333, + "P2_VT01": 0.4668916666666666, + "P2_VXT02": 0.9322583333333334, + "P2_VXT03": 0.9784166666666666, + "P2_VYT02": 0.5474958333333334, + "P2_VYT03": 0.5795333333333332, + "P3_FIT01": 0.3636125, + "P3_LCP01D": 0.57795, + "P3_LCV01D": 0.49150000000000005, + "P3_LIT01": 0.44431666666666664, + "P3_PIT01": 0.4088541666666667, + "P4_HT_FD": 0.5907875, + "P4_HT_LD": 0.06742500000000001, + "P4_HT_PO": 0.05970416666666667, + "P4_LD": 0.4144958333333334, + "P4_ST_FD": 0.168875, + "P4_ST_GOV": 0.4306625, + "P4_ST_LD": 0.51475, + "P4_ST_PO": 0.38827083333333334, + "P4_ST_PT01": 0.7711666666666667, + "P4_ST_TT01": 0.515625 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.02845, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0011099999999999999, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.11887999999999987, + "q25_diff": 0.012909999999999977, + "q50_diff": 0.0697000000000001, + "q75_diff": 0.026550000000000074, + "q95_diff": 0.04960999999999993 + }, + "P1_B3004": { + "q05_diff": 14.80313000000001, + "q25_diff": 1.6197799999999916, + "q50_diff": 6.726189999999974, + "q75_diff": 8.508819999999957, + "q95_diff": 37.86444 + }, + "P1_B3005": { + "q05_diff": 40.40789999999993, + "q25_diff": 8.045049999999947, + "q50_diff": 89.51202, + "q75_diff": 104.97393, + "q95_diff": 4.4593499999998585 + }, + "P1_B4002": { + "q05_diff": 0.4564699999999995, + "q25_diff": 1.6555000000000035, + "q50_diff": 1.6555000000000035, + "q75_diff": 0.9287000000000063, + "q95_diff": 0.16969000000000278 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 81.96193, + "q50_diff": 0.04000000000000625, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 2718.85966, + "q25_diff": 200.6357399999997, + "q50_diff": 399.82324000000017, + "q75_diff": 690.95606, + "q95_diff": 2133.9103999999998 + }, + "P1_B4022": { + "q05_diff": 0.38222999999999985, + "q25_diff": 1.3479000000000028, + "q50_diff": 1.4273799999999994, + "q75_diff": 0.9127399999999994, + "q95_diff": 0.5270299999999963 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.05340999999999996, + "q75_diff": 38.169869999999996, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 1.3654200000000003, + "q25_diff": 0.2554099999999977, + "q50_diff": 8.60333, + "q75_diff": 10.84863, + "q95_diff": 0.824300000000008 + }, + "P1_FCV03Z": { + "q05_diff": 1.7547599999999974, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.148199999999996, + "q75_diff": 10.864249999999998, + "q95_diff": 0.9307899999999876 + }, + "P1_FT01": { + "q05_diff": 120.72912000000001, + "q25_diff": 135.22494, + "q50_diff": 139.04176, + "q75_diff": 136.48351, + "q95_diff": 55.99644999999998 + }, + "P1_FT01Z": { + "q05_diff": 387.58875, + "q25_diff": 420.63922, + "q50_diff": 434.99024, + "q75_diff": 448.03643999999997, + "q95_diff": 136.61852999999996 + }, + "P1_FT02": { + "q05_diff": 1991.54888, + "q25_diff": 139.78015000000005, + "q50_diff": 286.57434999999987, + "q75_diff": 414.56177, + "q95_diff": 1577.1709 + }, + "P1_FT02Z": { + "q05_diff": 1.9862300000000026, + "q25_diff": 2300.54303, + "q50_diff": 14.318849999999657, + "q75_diff": 7.299800000000232, + "q95_diff": 6.317140000000109 + }, + "P1_FT03": { + "q05_diff": 166.6325, + "q25_diff": 161.89179, + "q50_diff": 130.77871, + "q75_diff": 112.34787, + "q95_diff": 67.71792999999997 + }, + "P1_FT03Z": { + "q05_diff": 23.41980000000001, + "q25_diff": 2.9805299999999306, + "q50_diff": 84.98266000000012, + "q75_diff": 100.6383699999999, + "q95_diff": 3.5079399999999623 + }, + "P1_LCV01D": { + "q05_diff": 0.17497000000000007, + "q25_diff": 0.29953000000000074, + "q50_diff": 4.171619999999999, + "q75_diff": 3.162939999999999, + "q95_diff": 0.7125800000000009 + }, + "P1_LCV01Z": { + "q05_diff": 0.0, + "q25_diff": 0.5874600000000001, + "q50_diff": 4.638670000000001, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.9536700000000007 + }, + "P1_LIT01": { + "q05_diff": 41.02197000000001, + "q25_diff": 39.912379999999985, + "q50_diff": 25.052059999999983, + "q75_diff": 11.556579999999997, + "q95_diff": 6.819880000000012 + }, + "P1_PCV01D": { + "q05_diff": 0.18625000000000114, + "q25_diff": 2.7490099999999984, + "q50_diff": 18.274499999999996, + "q75_diff": 43.266600000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.22888000000000375, + "q25_diff": 2.0370500000000007, + "q50_diff": 16.029360000000004, + "q75_diff": 43.33495, + "q95_diff": 45.1493 + }, + "P1_PCV02Z": { + "q05_diff": 0.2288800000000002, + "q25_diff": 0.2212499999999995, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_PIT01": { + "q05_diff": 0.12588, + "q25_diff": 0.09983000000000009, + "q50_diff": 0.14529999999999998, + "q75_diff": 0.28562, + "q95_diff": 0.27505 + }, + "P1_PIT02": { + "q05_diff": 0.3077, + "q25_diff": 0.09689999999999999, + "q50_diff": 0.029230000000000034, + "q75_diff": 0.09885, + "q95_diff": 0.29157999999999995 + }, + "P1_TIT01": { + "q05_diff": 1.1499400000000009, + "q25_diff": 1.0675299999999979, + "q50_diff": 0.7149500000000018, + "q75_diff": 0.24499999999999744, + "q95_diff": 0.0903199999999984 + }, + "P1_TIT02": { + "q05_diff": 1.5033499999999975, + "q25_diff": 1.9118800000000036, + "q50_diff": 1.714420000000004, + "q75_diff": 1.4456300000000013, + "q95_diff": 0.03227000000000402 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.00013999999999825263, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.00010999999999938836 + }, + "P2_CO_rpm": { + "q05_diff": 11.330000000001746, + "q25_diff": 8.69999999999709, + "q50_diff": 5.690000000002328, + "q75_diff": 7.30000000000291, + "q95_diff": 4.44999999999709 + }, + "P2_HILout": { + "q05_diff": 0.3295900000000529, + "q25_diff": 1.6540499999999838, + "q50_diff": 0.31737999999995736, + "q75_diff": 1.7334000000000742, + "q95_diff": 2.2399900000000343 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 7.360000000000014, + "q25_diff": 5.57000000000005, + "q50_diff": 5.840000000000032, + "q75_diff": 5.5, + "q95_diff": 3.3899999999999864 + }, + "P2_SIT02": { + "q05_diff": 7.619019999999978, + "q25_diff": 6.217650000000049, + "q50_diff": 6.172910000000002, + "q75_diff": 5.812379999999962, + "q95_diff": 5.001769999999965 + }, + "P2_VT01": { + "q05_diff": 0.08244000000000007, + "q25_diff": 0.10481000000000051, + "q50_diff": 0.05597999999999992, + "q75_diff": 0.0054800000000003735, + "q95_diff": 0.0011799999999997368 + }, + "P2_VXT02": { + "q05_diff": 0.9418600000000001, + "q25_diff": 1.1576499999999998, + "q50_diff": 0.8090000000000002, + "q75_diff": 0.51065, + "q95_diff": 0.5369000000000002 + }, + "P2_VXT03": { + "q05_diff": 1.15339, + "q25_diff": 1.1620499999999998, + "q50_diff": 0.9971699999999999, + "q75_diff": 0.83748, + "q95_diff": 1.09912 + }, + "P2_VYT02": { + "q05_diff": 0.41571999999999987, + "q25_diff": 0.9217900000000001, + "q50_diff": 0.6209599999999997, + "q75_diff": 0.2496299999999998, + "q95_diff": 0.2185100000000002 + }, + "P2_VYT03": { + "q05_diff": 0.057640000000000136, + "q25_diff": 0.4686199999999996, + "q50_diff": 0.46452000000000027, + "q75_diff": 0.2946100000000005, + "q95_diff": 0.28494999999999937 + }, + "P3_FIT01": { + "q05_diff": 928.66, + "q25_diff": 256.27, + "q50_diff": 1503.74, + "q75_diff": 754.9899999999998, + "q95_diff": 193.55000000000018 + }, + "P3_LCP01D": { + "q05_diff": 3916.0, + "q25_diff": 5120.0, + "q50_diff": 5198.0, + "q75_diff": 102.0, + "q95_diff": 2435.0 + }, + "P3_LCV01D": { + "q05_diff": 1945.0, + "q25_diff": 2748.0, + "q50_diff": 4739.0, + "q75_diff": 7291.0, + "q95_diff": 6816.0 + }, + "P3_LIT01": { + "q05_diff": 3192.3999999999996, + "q25_diff": 757.8799999999992, + "q50_diff": 1750.5100000000002, + "q75_diff": 3869.7700000000004, + "q95_diff": 5256.780000000001 + }, + "P3_PIT01": { + "q05_diff": 234.13, + "q25_diff": 124.66999999999999, + "q50_diff": 66.61, + "q75_diff": 702.55, + "q95_diff": 3113.87 + }, + "P4_HT_FD": { + "q05_diff": 0.0013199999999999998, + "q25_diff": 3.9999999999999996e-05, + "q50_diff": 0.0037099999999999998, + "q75_diff": 0.0044800000000000005, + "q95_diff": 0.0047599999999999995 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 27.14122, + "q75_diff": 0.13020999999999106, + "q95_diff": 0.21701000000000192 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 0.9223299999999999, + "q50_diff": 8.89752, + "q75_diff": 0.054199999999994475, + "q95_diff": 0.6510700000000043 + }, + "P4_LD": { + "q05_diff": 40.30498, + "q25_diff": 80.16645, + "q50_diff": 73.74402000000003, + "q75_diff": 50.643100000000004, + "q95_diff": 78.09792999999996 + }, + "P4_ST_FD": { + "q05_diff": 0.0007900000000000003, + "q25_diff": 0.0006099999999999999, + "q50_diff": 0.00058, + "q75_diff": 0.0001600000000000001, + "q95_diff": 0.0006099999999999999 + }, + "P4_ST_GOV": { + "q05_diff": 653.1599999999999, + "q25_diff": 1090.1299999999992, + "q50_diff": 1825.2299999999996, + "q75_diff": 659.5400000000009, + "q95_diff": 273.40999999999985 + }, + "P4_ST_LD": { + "q05_diff": 25.522340000000014, + "q25_diff": 54.95673000000002, + "q50_diff": 52.53005999999999, + "q75_diff": 30.317200000000014, + "q95_diff": 3.5045499999999947 + }, + "P4_ST_PO": { + "q05_diff": 16.012389999999982, + "q25_diff": 16.468050000000005, + "q50_diff": 30.65924000000001, + "q75_diff": 9.563509999999951, + "q95_diff": 11.301789999999983 + }, + "P4_ST_PT01": { + "q05_diff": 24.659999999999854, + "q25_diff": 55.17000000000007, + "q50_diff": 55.30999999999949, + "q75_diff": 101.72999999999956, + "q95_diff": 76.97999999999956 + }, + "P4_ST_TT01": { + "q05_diff": 50.419999999998254, + "q25_diff": 25.590000000000146, + "q50_diff": 1.5400000000008731, + "q75_diff": 7.9099999999998545, + "q95_diff": 12.450000000000728 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0624831607922858, + "P1_B2016": 0.005256433366776614, + "P1_B3004": 0.9420514424699565, + "P1_B3005": 0.9755864990985265, + "P1_B4002": 0.9939428775878422, + "P1_B4005": 1.0901635813805555, + "P1_B400B": 0.028092634631170177, + "P1_B4022": 1.0728287547128275, + "P1_FCV02Z": 1.0856923346443097, + "P1_FCV03D": 1.0287616800381814, + "P1_FCV03Z": 1.01920928123416, + "P1_FT01": 0.04946603703802366, + "P1_FT01Z": 0.05045622563091379, + "P1_FT02": 0.02664690203439979, + "P1_FT02Z": 1.0572315062387077, + "P1_FT03": 0.06818052959716525, + "P1_FT03Z": 0.9861265787637634, + "P1_LCV01D": 0.9922897795117445, + "P1_LCV01Z": 0.9035160861736835, + "P1_LIT01": 0.032560279359611255, + "P1_PCV01D": 0.9847169107083952, + "P1_PCV01Z": 1.1483081333334428, + "P1_PCV02Z": 0.5764046458797007, + "P1_PIT01": 0.0021008931037195255, + "P1_PIT02": 0.013277356227836434, + "P1_TIT01": 0.013176470128318685, + "P1_TIT02": 0.0066781343327798615, + "P2_24Vdc": 0.039965792976989475, + "P2_CO_rpm": 0.2763124243951074, + "P2_HILout": 0.24706059027478758, + "P2_MSD": 0.0, + "P2_SIT01": 0.3958485218427761, + "P2_SIT02": 0.5077622761106568, + "P2_VT01": 0.09799551246438576, + "P2_VXT02": 0.08617199848952684, + "P2_VXT03": 0.020537043704080604, + "P2_VYT02": 0.004882186781516085, + "P2_VYT03": 0.11977875042928843, + "P3_FIT01": 0.015510481846798818, + "P3_LCP01D": 0.028799148940010566, + "P3_LCV01D": 0.02734570443173201, + "P3_LIT01": 0.06146827249625708, + "P3_PIT01": 0.021691058833552712, + "P4_HT_FD": 0.48478468436597033, + "P4_HT_LD": 1.052071148480924, + "P4_HT_PO": 1.0918064404320895, + "P4_LD": 0.025261119593570136, + "P4_ST_FD": 0.17359710056794989, + "P4_ST_GOV": 0.009656930397496155, + "P4_ST_LD": 0.018143802080119564, + "P4_ST_PO": 0.008112295501195299, + "P4_ST_PT01": 0.003276191275868312, + "P4_ST_TT01": 0.015004036696998568 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.09144631115705412, + "P1_FCV02D": 0.07101077239149962, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_quantile_loss__seed1337/comprehensive_eval.json b/example/results/ablations/runs/no_quantile_loss__seed1337/comprehensive_eval.json new file mode 100644 index 0000000..6a0941b --- /dev/null +++ b/example/results/ablations/runs/no_quantile_loss__seed1337/comprehensive_eval.json @@ -0,0 +1,537 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_quantile_loss__seed1337/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.4130987421383648, + "avg_jsd": 0.017899328657243745, + "avg_lag1_diff": 0.3108961322076135 + }, + "two_sample": { + "continuous_mmd_rbf": 0.6504309177398682, + "continuous_mmd_gamma": 1.7152955179462124e-05, + "histogram_mmd_rbf": 0.6504309177398682, + "histogram_mmd_gamma": 1.715198981171805e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 162.71400451660156, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 2.948979736458853, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.21265330910682678, + "corr_frobenius": 16.46584129333496, + "lag1_corr_mean_abs_diff": 0.20872150361537933, + "lag1_corr_frobenius": 16.312320709228516, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.29016542434692383, + "corr_frobenius": 10.296795845031738, + "lag1_corr_mean_abs_diff": 0.30376631021499634, + "lag1_corr_frobenius": 10.575726509094238 + }, + "P2": { + "corr_mean_abs_diff": 0.14009949564933777, + "corr_frobenius": 2.959531307220459, + "lag1_corr_mean_abs_diff": 0.1557459831237793, + "lag1_corr_frobenius": 2.749875783920288 + }, + "P3": { + "corr_mean_abs_diff": 0.4514458477497101, + "corr_frobenius": 3.0256309509277344, + "lag1_corr_mean_abs_diff": 0.5530626773834229, + "lag1_corr_frobenius": 3.2009949684143066 + }, + "P4": { + "corr_mean_abs_diff": 0.17017430067062378, + "corr_frobenius": 2.646982192993164, + "lag1_corr_mean_abs_diff": 0.19284602999687195, + "lag1_corr_frobenius": 2.9210431575775146 + } + } + }, + "frequency": { + "avg_psd_l1": 0.02026984840631485, + "avg_psd_cosine": 0.36866448331273566, + "avg_low_high_ratio_abs_diff": 9433975.389639813 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 15.0, + "mean_dwell": 24.0, + "median_dwell": 2.5, + "mean_step": 68.42660395304362, + "median_step": 1.9043693542480469 + }, + "P1_B2004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B3004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.01115664489586733, + "median_step": 0.004810333251953125 + }, + "P1_B3005": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22998.14285714286, + "mean_abs_diff_mean_dwell": 14201.625685203559, + "mean_abs_diff_median_dwell": 14207.642857142857, + "mean_abs_diff_mean_step": 33.693117622162895, + "mean_abs_diff_median_step": 0.831573486328125 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 1.0, + "change_rate": 0.0, + "step_median": NaN + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.411041259765625, + "mean_abs_diff_change_rate": 0.4608201173921447 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.0026041666666666665, + "top1_mass": 1.0, + "top3_mass": 1.0, + "median_dwell": 384.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8880208333333334, + "top1_mass": 0.028645833333333332, + "top3_mass": 0.078125, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9192708333333334, + "top1_mass": 0.08333333333333333, + "top3_mass": 0.08854166666666667, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.033854166666666664, + "top1_mass": 0.4010416666666667, + "top3_mass": 0.6927083333333334, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.9947916666666666, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.013020833333333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4036458333333333, + "top1_mass": 0.09114583333333333, + "top3_mass": 0.1796875, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.9713541666666666, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.015625, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.19791666666666666, + "top1_mass": 0.125, + "top3_mass": 0.3541666666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5256792704264323, + "mean_abs_diff_top1_mass": 0.1047375996907552, + "mean_abs_diff_top3_mass": 0.08982976277669272, + "mean_abs_diff_median_dwell": 47.75 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17409999668598175, + "q50": 0.17409999668598175, + "q95": 0.27557000517845154, + "tail_ratio": NaN + }, + "P2_SIT02": { + "q05": 774.9180908203125, + "q50": 784.552734375, + "q95": 791.6948852539062, + "tail_ratio": 0.7412989217884881 + }, + "P1_FT03": { + "q05": 209.21278381347656, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.19314856245818826 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.672994390130043, + "mean_abs_diff_q50": 2.267071540157, + "mean_abs_diff_q95": 3.5562773644924164, + "mean_abs_diff_tail_ratio": 0.04647519067886792 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 381.0, + "mean_dwell": 1.0052356020942408, + "median_dwell": 1.0, + "mean_step": 15.895563080554872, + "median_step": 5.94061279296875 + }, + "P1_FT02Z": { + "num_changes": 382.0, + "mean_dwell": 1.0026109660574412, + "median_dwell": 1.0, + "mean_step": 747.9839405364391, + "median_step": 305.9484672546387 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89281.0, + "mean_abs_diff_mean_dwell": 0.09700721625641795, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 374.2886512329816, + "mean_abs_diff_median_step": 152.67763328552246 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 33.835411071777344, + "std": 31.693500518798828, + "lag1": 0.08183889836072922 + }, + "P2_24Vdc": { + "mean": 28.028623580932617, + "std": 0.003516798373311758, + "lag1": 0.1838952898979187 + }, + "P2_HILout": { + "mean": 711.4879760742188, + "std": 9.422582626342773, + "lag1": 0.08547315746545792 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.3363494873046875, + "mean_abs_diff_std": 0.5190210201932738, + "mean_abs_diff_lag1": 0.40582848712801933 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9654670357704163, + "mae": 0.779840350151062 + }, + "real_plus_synthetic": { + "rmse": 0.5558143258094788, + "mae": 0.36008939146995544 + }, + "rmse_ratio_synth_to_real": 1.7291402234986348 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8323747680890538, + "auprc": 0.5904586531962263, + "threshold": 0.004217079374939203, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6666666666666667, + "best_f1_threshold": 1.43533456325531 + }, + "real_plus_synthetic": { + "auroc": 0.8787801484230056, + "auprc": 0.6451654579890704, + "threshold": 0.3155173659324646, + "f1": 0.2591093117408907, + "precision": 0.15763546798029557, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.5558031797409058 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_quantile_loss__seed1337/comprehensive_eval_post.json b/example/results/ablations/runs/no_quantile_loss__seed1337/comprehensive_eval_post.json new file mode 100644 index 0000000..e2eefc7 --- /dev/null +++ b/example/results/ablations/runs/no_quantile_loss__seed1337/comprehensive_eval_post.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_quantile_loss__seed1337/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.19799482822418213, + "continuous_mmd_gamma": 5.034502072012259e-05, + "histogram_mmd_rbf": 0.1980680227279663, + "histogram_mmd_gamma": 5.033049044703699e-05, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 89.32330322265625, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 1.6188687137271414, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.1945081353187561, + "corr_frobenius": 16.07838249206543, + "lag1_corr_mean_abs_diff": 0.20114566385746002, + "lag1_corr_frobenius": 16.6044921875, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.33709853887557983, + "corr_frobenius": 11.78930377960205, + "lag1_corr_mean_abs_diff": 0.3601304590702057, + "lag1_corr_frobenius": 12.476584434509277 + }, + "P2": { + "corr_mean_abs_diff": 0.1022777333855629, + "corr_frobenius": 2.231959581375122, + "lag1_corr_mean_abs_diff": 0.15117810666561127, + "lag1_corr_frobenius": 2.7848243713378906 + }, + "P3": { + "corr_mean_abs_diff": 0.4514458477497101, + "corr_frobenius": 3.0256309509277344, + "lag1_corr_mean_abs_diff": 0.5530626773834229, + "lag1_corr_frobenius": 3.2009949684143066 + }, + "P4": { + "corr_mean_abs_diff": 0.2023613303899765, + "corr_frobenius": 3.1925668716430664, + "lag1_corr_mean_abs_diff": 0.2375580072402954, + "lag1_corr_frobenius": 3.5891406536102295 + } + } + }, + "frequency": { + "avg_psd_l1": 0.022722473368048668, + "avg_psd_cosine": 0.4490051342631286, + "avg_low_high_ratio_abs_diff": 18.416101711948972 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 308.0, + "mean_dwell": 1.2427184466019416, + "median_dwell": 1.0, + "mean_step": 0.8413416007896523, + "median_step": 0.7274894714355469 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 347.0, + "mean_dwell": 1.103448275862069, + "median_dwell": 1.0, + "mean_step": 41.3141472878254, + "median_step": 54.116028785705566 + }, + "P1_B2004": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 0.014068839402653235, + "median_step": 0.0017450004816055298 + }, + "P1_B3004": { + "num_changes": 370.0, + "mean_dwell": 1.0350404312668464, + "median_dwell": 1.0, + "mean_step": 14.77256890374261, + "median_step": 10.804779052734375 + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.8157603410765646, + "median_step": 0.5944099426269531 + }, + "P1_B3005": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 64.28072175926926, + "median_step": 41.228179931640625 + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22997.85714285714, + "mean_abs_diff_mean_dwell": 14417.281142645239, + "mean_abs_diff_median_dwell": 13988.57142857143, + "mean_abs_diff_mean_step": 19.03385039702909, + "mean_abs_diff_median_step": 16.840154950817425 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6822916666666666, + "change_rate": 0.7702349869451697, + "step_median": 68.81373023986816 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.09333292643229163, + "mean_abs_diff_change_rate": 0.309414869553025, + "mean_abs_diff_step_median": 68.78170993924141 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.015625, + "top1_mass": 0.40625, + "top3_mass": 0.7760416666666666, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8776041666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.059895833333333336, + "top3_mass": 0.0703125, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.18229166666666666, + "top3_mass": 0.3697916666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.8463541666666666, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4817708333333333, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.7864583333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3515625, + "top1_mass": 0.036458333333333336, + "top3_mass": 0.09635416666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5303484598795573, + "mean_abs_diff_top1_mass": 0.03112411499023438, + "mean_abs_diff_top3_mass": 0.05629857381184896, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17409999668598175, + "q50": 0.17409999668598175, + "q95": 0.27557000517845154, + "tail_ratio": NaN + }, + "P2_SIT02": { + "q05": 774.9180908203125, + "q50": 784.552734375, + "q95": 791.6948852539062, + "tail_ratio": 0.7412989217884881 + }, + "P1_FT03": { + "q05": 209.21278381347656, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.19314856245818826 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.672994390130043, + "mean_abs_diff_q50": 2.267071540157, + "mean_abs_diff_q95": 3.5562773644924164, + "mean_abs_diff_tail_ratio": 0.04647519067886792 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 64.74391126130756, + "median_step": 62.410614013671875 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 1312.3134692358592, + "median_step": 1284.6439208984375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.09039718638964156, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 680.877589673068, + "mean_abs_diff_median_step": 670.2603607177734 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 35.254512786865234, + "std": 31.650897979736328, + "lag1": -0.0927804708480835 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.003297113813459873, + "lag1": 0.06643600016832352 + }, + "P2_HILout": { + "mean": 711.4987182617188, + "std": 7.856828212738037, + "lag1": -0.013758387416601181 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.4565467834472656, + "mean_abs_diff_std": 0.2373291968833655, + "mean_abs_diff_lag1": 0.4579590285817782 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9617411494255066, + "mae": 0.7749478816986084 + }, + "real_plus_synthetic": { + "rmse": 0.5492608547210693, + "mae": 0.3524129092693329 + }, + "rmse_ratio_synth_to_real": 1.7224672044224043 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8285482374768088, + "auprc": 0.589142869210671, + "threshold": 0.006542472168803215, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6666666666666667, + "best_f1_threshold": 1.4494516849517822 + }, + "real_plus_synthetic": { + "auroc": 0.8857838589981447, + "auprc": 0.6447892653801922, + "threshold": 0.3054894804954529, + "f1": 0.25196850393700787, + "precision": 0.1523809523809524, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.5378162860870361 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_quantile_loss__seed1337/eval.json b/example/results/ablations/runs/no_quantile_loss__seed1337/eval.json new file mode 100644 index 0000000..90ec3bc --- /dev/null +++ b/example/results/ablations/runs/no_quantile_loss__seed1337/eval.json @@ -0,0 +1,974 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.11384, + "std": 0.0 + }, + "P1_B2016": { + "mean": 1.3915577083333348, + "std": 0.13947614022053412 + }, + "P1_B3004": { + "mean": 396.00211, + "std": 0.0 + }, + "P1_B3005": { + "mean": 1015.43976, + "std": 0.0 + }, + "P1_B4002": { + "mean": 33.54375, + "std": 0.0 + }, + "P1_B4005": { + "mean": 0.0, + "std": 0.0 + }, + "P1_B400B": { + "mean": 1772.6605813802094, + "std": 1195.5247571148332 + }, + "P1_B4022": { + "mean": 36.881583541666664, + "std": 0.14105102171131376 + }, + "P1_FCV02Z": { + "mean": 22.643384531250003, + "std": 40.62085789957324 + }, + "P1_FCV03D": { + "mean": 58.57125942708336, + "std": 6.64094608745763 + }, + "P1_FCV03Z": { + "mean": 59.044430703124966, + "std": 6.817517409732479 + }, + "P1_FT01": { + "mean": 194.93169489583332, + "std": 31.907421147789986 + }, + "P1_FT01Z": { + "mean": 877.3088108593755, + "std": 84.42922242684735 + }, + "P1_FT02": { + "mean": 1130.398466666666, + "std": 891.1246863991283 + }, + "P1_FT02Z": { + "mean": 1130.398466666666, + "std": 891.1246863991283 + }, + "P1_FT03": { + "mean": 282.4349720572918, + "std": 33.64962542238899 + }, + "P1_FT03Z": { + "mean": 282.4349720572918, + "std": 33.64962542238899 + }, + "P1_LCV01D": { + "mean": 12.16683747395832, + "std": 3.0449963797735586 + }, + "P1_LCV01Z": { + "mean": 11.635974218749997, + "std": 3.303646728913521 + }, + "P1_LIT01": { + "mean": 404.03647182291667, + "std": 15.824286512578475 + }, + "P1_PCV01D": { + "mean": 67.45003744791663, + "std": 23.85186428670996 + }, + "P1_PCV01Z": { + "mean": 68.276033671875, + "std": 23.933764405919202 + }, + "P1_PCV02Z": { + "mean": 11.76605, + "std": 0.0 + }, + "P1_PIT01": { + "mean": 1.366327500000001, + "std": 0.0673200977482619 + }, + "P1_PIT02": { + "mean": 0.22190658854166648, + "std": 0.05073903510741829 + }, + "P1_TIT01": { + "mean": 35.65866359374994, + "std": 0.5836675616735701 + }, + "P1_TIT02": { + "mean": 36.50122700520829, + "std": 1.2332879905119984 + }, + "P2_24Vdc": { + "mean": 28.028623359374986, + "std": 0.003516726375929793 + }, + "P2_CO_rpm": { + "mean": 54102.283833828114, + "std": 24.087364527416046 + }, + "P2_HILout": { + "mean": 711.4879614583338, + "std": 9.422582442545936 + }, + "P2_MSD": { + "mean": 758.4425, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.2531249999992, + "std": 5.8591930957840725 + }, + "P2_SIT02": { + "mean": 783.923604296875, + "std": 5.842625008175601 + }, + "P2_VT01": { + "mean": 11.923782578124996, + "std": 0.06898207207491282 + }, + "P2_VXT02": { + "mean": -3.2992371614583327, + "std": 0.4773813056787038 + }, + "P2_VXT03": { + "mean": -1.3056961718749989, + "std": 0.3573689267646248 + }, + "P2_VYT02": { + "mean": 3.9808476822916674, + "std": 0.5056978789649539 + }, + "P2_VYT03": { + "mean": 6.233340729166663, + "std": 0.38920688744478077 + }, + "P3_FIT01": { + "mean": 888.4507813802076, + "std": 1676.1943844854939 + }, + "P3_LCP01D": { + "mean": 4590.880697031249, + "std": 5143.4225118731565 + }, + "P3_LCV01D": { + "mean": 7134.490903958336, + "std": 6860.174505041899 + }, + "P3_LIT01": { + "mean": 13352.644031640628, + "std": 4370.946869247855 + }, + "P3_PIT01": { + "mean": 392.9633114062501, + "std": 988.8695443670978 + }, + "P4_HT_FD": { + "mean": -0.0004982812500000002, + "std": 0.0044673488687889595 + }, + "P4_HT_LD": { + "mean": -115.69799739583333, + "std": 35.27327640165136 + }, + "P4_HT_PO": { + "mean": 33.83541208333332, + "std": 31.693500790680886 + }, + "P4_LD": { + "mean": 366.12485794270833, + "std": 61.02942352172124 + }, + "P4_ST_FD": { + "mean": -6.640625000000009e-05, + "std": 0.004044489801718016 + }, + "P4_ST_GOV": { + "mean": 17918.738952656262, + "std": 1897.4704889747775 + }, + "P4_ST_LD": { + "mean": 332.09368789062484, + "std": 38.593957056472306 + }, + "P4_ST_PO": { + "mean": 332.36840252604156, + "std": 35.260567478365545 + }, + "P4_ST_PT01": { + "mean": 10045.100718177076, + "std": 19.20451867005973 + }, + "P4_ST_TT01": { + "mean": 27608.458333333358, + "std": 24.483240568921808 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.020272347909024324, + "std_abs_err": 0.017998437082092405 + }, + "P1_B2016": { + "mean_abs_err": 0.00029138366856784614, + "std_abs_err": 0.003702577743971036 + }, + "P1_B3004": { + "mean_abs_err": 7.383953824526145, + "std_abs_err": 15.41814958804181 + }, + "P1_B3005": { + "mean_abs_err": 45.609026528629784, + "std_abs_err": 61.18331651100028 + }, + "P1_B4002": { + "mean_abs_err": 1.2268236801201198, + "std_abs_err": 0.5786734991990778 + }, + "P1_B4005": { + "mean_abs_err": 60.52510701619792, + "std_abs_err": 42.18129744770969 + }, + "P1_B400B": { + "mean_abs_err": 5.81528879775874, + "std_abs_err": 2.1724841957604895 + }, + "P1_B4022": { + "mean_abs_err": 0.8204088942641405, + "std_abs_err": 0.5303998331235046 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.307550369222273, + "std_abs_err": 0.18237246031505805 + }, + "P1_FCV03D": { + "mean_abs_err": 0.0033959762365327606, + "std_abs_err": 0.0414503687488752 + }, + "P1_FCV03Z": { + "mean_abs_err": 0.25460847706487044, + "std_abs_err": 0.10271344540087668 + }, + "P1_FT01": { + "mean_abs_err": 0.20804053955305335, + "std_abs_err": 4.5594393091456595 + }, + "P1_FT01Z": { + "mean_abs_err": 1.7408706447770328, + "std_abs_err": 8.663519463529099 + }, + "P1_FT02": { + "mean_abs_err": 2.4196865679600705, + "std_abs_err": 1.9285225477157155 + }, + "P1_FT02Z": { + "mean_abs_err": 648.0799799419826, + "std_abs_err": 302.22066937202123 + }, + "P1_FT03": { + "mean_abs_err": 0.20399821893710168, + "std_abs_err": 0.17541231109348843 + }, + "P1_FT03Z": { + "mean_abs_err": 778.6070662912298, + "std_abs_err": 27.475863441228334 + }, + "P1_LCV01D": { + "mean_abs_err": 0.012504812183184555, + "std_abs_err": 0.13038142674343733 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.3300998105879209, + "std_abs_err": 0.34017723156820967 + }, + "P1_LIT01": { + "mean_abs_err": 0.07840034008961538, + "std_abs_err": 0.1923884642045035 + }, + "P1_PCV01D": { + "mean_abs_err": 0.01553628777475069, + "std_abs_err": 0.08209758580438731 + }, + "P1_PCV01Z": { + "mean_abs_err": 0.13211812502758846, + "std_abs_err": 0.06916459525555751 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.07670294660492694, + "std_abs_err": 0.09684446546464737 + }, + "P1_PIT01": { + "mean_abs_err": 0.011153720753390939, + "std_abs_err": 0.005887538378379804 + }, + "P1_PIT02": { + "mean_abs_err": 0.2652505017998502, + "std_abs_err": 0.44797260294746155 + }, + "P1_TIT01": { + "mean_abs_err": 0.022544648683272328, + "std_abs_err": 0.0003869344876211711 + }, + "P1_TIT02": { + "mean_abs_err": 0.023751673128238338, + "std_abs_err": 0.011529264487735391 + }, + "P2_24Vdc": { + "mean_abs_err": 8.634325268275234e-05, + "std_abs_err": 0.00017426026152287787 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.426364054561418, + "std_abs_err": 1.6316327526276808 + }, + "P2_HILout": { + "mean_abs_err": 0.02515218823759824, + "std_abs_err": 0.9678565652897735 + }, + "P2_MSD": { + "mean_abs_err": 4.750739999999951, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.6507924134438099, + "std_abs_err": 0.6602777353398475 + }, + "P2_SIT02": { + "mean_abs_err": 0.019866414328816973, + "std_abs_err": 0.652045251362634 + }, + "P2_VT01": { + "mean_abs_err": 3.3067885137683106e-05, + "std_abs_err": 0.00046839000291075406 + }, + "P2_VXT02": { + "mean_abs_err": 0.0009413109316636081, + "std_abs_err": 0.0063483794074835065 + }, + "P2_VXT03": { + "mean_abs_err": 0.001197114789678988, + "std_abs_err": 0.0077415476009711215 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006211677012695205, + "std_abs_err": 0.005787425972166027 + }, + "P2_VYT03": { + "mean_abs_err": 0.0010643836312214106, + "std_abs_err": 0.009403925473689179 + }, + "P3_FIT01": { + "mean_abs_err": 262.4493947259756, + "std_abs_err": 103.53031330478098 + }, + "P3_LCP01D": { + "mean_abs_err": 6.153548790674904, + "std_abs_err": 13.422514157277874 + }, + "P3_LCV01D": { + "mean_abs_err": 0.614186977094505, + "std_abs_err": 20.246609226077453 + }, + "P3_LIT01": { + "mean_abs_err": 2.2004610533585947, + "std_abs_err": 16.772232319918658 + }, + "P3_PIT01": { + "mean_abs_err": 256.3945916170776, + "std_abs_err": 162.05238634009754 + }, + "P4_HT_FD": { + "mean_abs_err": 0.00039509330464826046, + "std_abs_err": 0.002435330554384859 + }, + "P4_HT_LD": { + "mean_abs_err": 149.69291447551893, + "std_abs_err": 1.8494828916530537 + }, + "P4_HT_PO": { + "mean_abs_err": 0.1483957114235892, + "std_abs_err": 0.2526295733594104 + }, + "P4_LD": { + "mean_abs_err": 0.0038958514859928073, + "std_abs_err": 0.4136171522782348 + }, + "P4_ST_FD": { + "mean_abs_err": 3.729302011685127e-05, + "std_abs_err": 0.002402313543727987 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.777691408737155, + "std_abs_err": 45.32727298700411 + }, + "P4_ST_LD": { + "mean_abs_err": 0.09079072080714923, + "std_abs_err": 0.7407877648547583 + }, + "P4_ST_PO": { + "mean_abs_err": 0.0994312023827888, + "std_abs_err": 0.8096239759572441 + }, + "P4_ST_PT01": { + "mean_abs_err": 5.474438642006135, + "std_abs_err": 1.4197486447406007 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.5743583461735398, + "std_abs_err": 0.56591876435348 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 1.0, + "P1_B2016": 0.043591666666666584, + "P1_B3004": 0.8108, + "P1_B3005": 0.8106, + "P1_B4002": 0.81055, + "P1_B4005": 0.9113, + "P1_B400B": 0.46663333333333334, + "P1_B4022": 0.5730333333333334, + "P1_FCV02Z": 0.6756416666666666, + "P1_FCV03D": 0.48804166666666665, + "P1_FCV03Z": 0.4923916666666667, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4746, + "P1_FT02": 0.4630875, + "P1_FT02Z": 0.8361000000000001, + "P1_FT03": 0.5241, + "P1_FT03Z": 1.0, + "P1_LCV01D": 0.44713749999999997, + "P1_LCV01Z": 0.4525416666666667, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5571083333333333, + "P1_PCV01Z": 0.5588041666666668, + "P1_PCV02Z": 1.0, + "P1_PIT01": 0.15849999999999997, + "P1_PIT02": 0.61435, + "P1_TIT01": 0.10056666666666658, + "P1_TIT02": 0.10090416666666668, + "P2_24Vdc": 0.11608333333333332, + "P2_CO_rpm": 0.09535833333333332, + "P2_HILout": 0.08234999999999998, + "P2_MSD": 1.0, + "P2_SIT01": 0.4783083333333333, + "P2_SIT02": 0.5016666666666667, + "P2_VT01": 0.12292083333333337, + "P2_VXT02": 0.2814458333333334, + "P2_VXT03": 0.43662083333333335, + "P2_VYT02": 0.250625, + "P2_VYT03": 0.26955416666666665, + "P3_FIT01": 0.22947499999999998, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.07123333333333334, + "P3_LIT01": 0.09349166666666664, + "P3_PIT01": 0.6393375, + "P4_HT_FD": 0.148725, + "P4_HT_LD": 0.9375, + "P4_HT_PO": 0.17305833333333331, + "P4_LD": 0.031137499999999985, + "P4_ST_FD": 0.028475000000000028, + "P4_ST_GOV": 0.03517083333333337, + "P4_ST_LD": 0.03240416666666668, + "P4_ST_PO": 0.035720833333333424, + "P4_ST_PT01": 0.38136666666666663, + "P4_ST_TT01": 0.06839166666666663 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.08333, + "q25_diff": 0.012630000000000002, + "q50_diff": 0.012630000000000002, + "q75_diff": 0.012630000000000002, + "q95_diff": 0.012630000000000002 + }, + "P1_B2016": { + "q05_diff": 0.011949999999999905, + "q25_diff": 0.010229999999999961, + "q50_diff": 0.005139999999999922, + "q75_diff": 0.014319999999999888, + "q95_diff": 0.029889999999999972 + }, + "P1_B3004": { + "q05_diff": 2.8398500000000126, + "q25_diff": 1.6357399999999984, + "q50_diff": 1.6357399999999984, + "q75_diff": 1.6357399999999984, + "q95_diff": 1.6357399999999984 + }, + "P1_B3005": { + "q05_diff": 13.44177000000002, + "q25_diff": 13.44177000000002, + "q50_diff": 13.44177000000002, + "q75_diff": 13.44177000000002, + "q95_diff": 99.67999000000009 + }, + "P1_B4002": { + "q05_diff": 1.5437500000000028, + "q25_diff": 0.11175000000000068, + "q50_diff": 0.11175000000000068, + "q75_diff": 0.11175000000000068, + "q95_diff": 0.11175000000000068 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 100.0, + "q50_diff": 100.0, + "q75_diff": 100.0, + "q95_diff": 100.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2479.8766800000003, + "q50_diff": 284.0803299999998, + "q75_diff": 9.545890000000327, + "q95_diff": 7.159419999999955 + }, + "P1_B4022": { + "q05_diff": 1.2294699999999992, + "q25_diff": 0.1574500000000043, + "q50_diff": 0.4875299999999996, + "q75_diff": 0.5946499999999943, + "q95_diff": 0.6461800000000011 + }, + "P1_FCV02Z": { + "q05_diff": 0.015249999999999986, + "q25_diff": 0.007630000000000026, + "q50_diff": 0.007630000000000026, + "q75_diff": 35.163889999999995, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 2.8725199999999944, + "q25_diff": 0.0981699999999961, + "q50_diff": 8.54466, + "q75_diff": 11.156169999999996, + "q95_diff": 0.6973699999999923 + }, + "P1_FCV03Z": { + "q05_diff": 3.517150000000001, + "q25_diff": 0.022889999999996746, + "q50_diff": 8.96454, + "q75_diff": 11.566150000000007, + "q95_diff": 0.5950900000000132 + }, + "P1_FT01": { + "q05_diff": 8.088040000000007, + "q25_diff": 8.587160000000011, + "q50_diff": 32.806329999999974, + "q75_diff": 37.95618999999999, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 38.51202999999998, + "q25_diff": 19.571230000000014, + "q50_diff": 74.79840000000002, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1898.00245, + "q50_diff": 258.24926000000005, + "q75_diff": 12.970209999999952, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 27.82217, + "q25_diff": 2759.94324, + "q50_diff": 1117.82361, + "q75_diff": 870.8322700000001, + "q95_diff": 865.2337600000001 + }, + "P1_FT03": { + "q05_diff": 35.89179999999999, + "q25_diff": 1.5264000000000237, + "q50_diff": 51.52633000000003, + "q75_diff": 59.766309999999976, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 788.6256599999999, + "q25_diff": 752.6723099999999, + "q50_diff": 703.89301, + "q75_diff": 697.6060500000001, + "q95_diff": 800.17028 + }, + "P1_LCV01D": { + "q05_diff": 0.1075399999999993, + "q25_diff": 1.206240000000001, + "q50_diff": 4.63275, + "q75_diff": 3.3112699999999986, + "q95_diff": 0.6183300000000003 + }, + "P1_LCV01Z": { + "q05_diff": 0.11345000000000027, + "q25_diff": 0.8315999999999999, + "q50_diff": 4.978860000000001, + "q75_diff": 3.2196099999999994, + "q95_diff": 0.7171600000000016 + }, + "P1_LIT01": { + "q05_diff": 14.39278999999999, + "q25_diff": 0.6645500000000197, + "q50_diff": 6.283670000000029, + "q75_diff": 8.229829999999993, + "q95_diff": 34.78140000000002 + }, + "P1_PCV01D": { + "q05_diff": 1.0922100000000015, + "q25_diff": 3.715759999999996, + "q50_diff": 22.261809999999997, + "q75_diff": 44.86057, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 1.0367800000000003, + "q25_diff": 3.5705600000000004, + "q50_diff": 22.777769999999997, + "q75_diff": 45.46356000000001, + "q95_diff": 45.14175 + }, + "P1_PCV02Z": { + "q05_diff": 0.2441399999999998, + "q25_diff": 0.2441399999999998, + "q50_diff": 0.2517700000000005, + "q75_diff": 0.2517700000000005, + "q95_diff": 0.2517700000000005 + }, + "P1_PIT01": { + "q05_diff": 0.01373000000000002, + "q25_diff": 0.008400000000000185, + "q50_diff": 0.010680000000000023, + "q75_diff": 0.02364999999999995, + "q95_diff": 0.008389999999999898 + }, + "P1_PIT02": { + "q05_diff": 0.03051999999999999, + "q25_diff": 0.10146999999999998, + "q50_diff": 0.10223999999999997, + "q75_diff": 0.0015300000000000313, + "q95_diff": 0.0022900000000000142 + }, + "P1_TIT01": { + "q05_diff": 0.015250000000001762, + "q25_diff": 0.06103999999999843, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.24414999999999765, + "q95_diff": 0.061029999999995255 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.030520000000002767, + "q50_diff": 0.09154999999999802, + "q75_diff": 0.30516999999999683, + "q95_diff": 0.4272400000000047 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.0008799999999986596, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.000920000000000698 + }, + "P2_CO_rpm": { + "q05_diff": 10.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 3.0, + "q95_diff": 0.0 + }, + "P2_HILout": { + "q05_diff": 0.619389999999953, + "q25_diff": 2.0173899999999776, + "q50_diff": 0.09057999999993172, + "q75_diff": 1.526740000000018, + "q95_diff": 1.1521599999999808 + }, + "P2_MSD": { + "q05_diff": 4.750739999999951, + "q25_diff": 4.750739999999951, + "q50_diff": 4.750739999999951, + "q75_diff": 4.750739999999951, + "q95_diff": 4.750739999999951 + }, + "P2_SIT01": { + "q05_diff": 1.0, + "q25_diff": 3.0, + "q50_diff": 5.0, + "q75_diff": 6.0, + "q95_diff": 6.0 + }, + "P2_SIT02": { + "q05_diff": 2.6579000000000406, + "q25_diff": 3.569340000000011, + "q50_diff": 5.617490000000089, + "q75_diff": 6.36414000000002, + "q95_diff": 6.8431399999999485 + }, + "P2_VT01": { + "q05_diff": 0.008470000000000866, + "q25_diff": 0.015229999999998967, + "q50_diff": 0.014129999999999754, + "q75_diff": 0.013349999999999085, + "q95_diff": 0.01886999999999972 + }, + "P2_VXT02": { + "q05_diff": 0.1316500000000005, + "q25_diff": 0.2994399999999997, + "q50_diff": 0.28542999999999985, + "q75_diff": 0.26605000000000034, + "q95_diff": 0.4595699999999998 + }, + "P2_VXT03": { + "q05_diff": 0.10078999999999994, + "q25_diff": 0.26920999999999995, + "q50_diff": 0.29757999999999996, + "q75_diff": 0.32704, + "q95_diff": 0.61108 + }, + "P2_VYT02": { + "q05_diff": 0.0950899999999999, + "q25_diff": 0.26207, + "q50_diff": 0.2269300000000003, + "q75_diff": 0.23554999999999993, + "q95_diff": 0.3890600000000006 + }, + "P2_VYT03": { + "q05_diff": 0.05600999999999967, + "q25_diff": 0.18581000000000003, + "q50_diff": 0.1761499999999998, + "q75_diff": 0.18344999999999967, + "q95_diff": 0.3527800000000001 + }, + "P3_FIT01": { + "q05_diff": 3.0, + "q25_diff": 2.0, + "q50_diff": 72.0, + "q75_diff": 2484.0, + "q95_diff": 18.942869999999857 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 101.40417000000002, + "q75_diff": 135.01952999999958, + "q95_diff": 8.21484000000055 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 256.0, + "q50_diff": 3226.9873, + "q75_diff": 174.61621000000014, + "q95_diff": 31.333010000000286 + }, + "P3_LIT01": { + "q05_diff": 1132.9770500000004, + "q25_diff": 34.37890999999945, + "q50_diff": 647.4140599999992, + "q75_diff": 1151.4589799999994, + "q95_diff": 1.6015599999991537 + }, + "P3_PIT01": { + "q05_diff": 1.0, + "q25_diff": 2.0, + "q50_diff": 3.0, + "q75_diff": 508.0, + "q95_diff": 188.0 + }, + "P4_HT_FD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 7e-05, + "q95_diff": 0.00021999999999999993 + }, + "P4_HT_LD": { + "q05_diff": 124.7862, + "q25_diff": 124.7862, + "q50_diff": 179.53157, + "q75_diff": 193.36982, + "q95_diff": 56.57957999999999 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 1.22976, + "q50_diff": 16.963260000000002, + "q75_diff": 0.8120300000000071, + "q95_diff": 0.08271000000000583 + }, + "P4_LD": { + "q05_diff": 0.5004900000000134, + "q25_diff": 3.1075700000000097, + "q50_diff": 3.4702399999999898, + "q75_diff": 0.07668999999998505, + "q95_diff": 4.30313000000001 + }, + "P4_ST_FD": { + "q05_diff": 6.999999999999975e-05, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 8.000000000000021e-05 + }, + "P4_ST_GOV": { + "q05_diff": 116.37305000000015, + "q25_diff": 38.04102000000057, + "q50_diff": 29.617190000000846, + "q75_diff": 109.16602000000057, + "q95_diff": 404.4824200000003 + }, + "P4_ST_LD": { + "q05_diff": 0.5155600000000504, + "q25_diff": 0.5112599999999929, + "q50_diff": 0.3724400000000401, + "q75_diff": 2.00551999999999, + "q95_diff": 7.728729999999985 + }, + "P4_ST_PO": { + "q05_diff": 2.1687599999999634, + "q25_diff": 0.8120700000000056, + "q50_diff": 0.5768199999999979, + "q75_diff": 2.1424200000000155, + "q95_diff": 7.583590000000015 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 20.0, + "q50_diff": 1.0, + "q75_diff": 1.0, + "q95_diff": 30.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 7.0, + "q50_diff": 3.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0189476811306442e-05, + "P1_B2016": 0.07618695239144935, + "P1_B3004": 1.018740504876714e-05, + "P1_B3005": 0.9999898125942444, + "P1_B4002": 0.9999898146591784, + "P1_B4005": 0.9999997508214005, + "P1_B400B": 0.45045912562159773, + "P1_B4022": 0.03037469907643997, + "P1_FCV02Z": 0.18182475630780803, + "P1_FCV03D": 0.3188790100307244, + "P1_FCV03Z": 0.1477047384515312, + "P1_FT01": 0.5820973686096496, + "P1_FT01Z": 0.3340826131634087, + "P1_FT02": 0.75191076012623, + "P1_FT02Z": 0.7519657387361416, + "P1_FT03": 0.2834155903958254, + "P1_FT03Z": 0.28327853335121855, + "P1_LCV01D": 0.14527008634204575, + "P1_LCV01Z": 0.21374529012207777, + "P1_LIT01": 0.12178882881233055, + "P1_PCV01D": 0.2508309265909373, + "P1_PCV01Z": 0.12279220909207755, + "P1_PCV02Z": 0.5669400491575541, + "P1_PIT01": 0.22468855469933502, + "P1_PIT02": 0.06124155098334361, + "P1_TIT01": 0.04468432299948122, + "P1_TIT02": 0.06696073912814682, + "P2_24Vdc": 0.15743657200974814, + "P2_CO_rpm": 0.34346874863740007, + "P2_HILout": 0.14782901011007804, + "P2_MSD": 0.0, + "P2_SIT01": 0.3875131407531049, + "P2_SIT02": 0.4052675740532808, + "P2_VT01": 0.14773170910846045, + "P2_VXT02": 0.163033385889365, + "P2_VXT03": 0.18198369820033933, + "P2_VYT02": 0.1452756215000487, + "P2_VYT03": 0.16882683150174038, + "P3_FIT01": 0.2716177197527262, + "P3_LCP01D": 0.5618088756774389, + "P3_LCV01D": 0.43449899105168155, + "P3_LIT01": 0.07644556223581167, + "P3_PIT01": 0.8737468921804842, + "P4_HT_FD": 0.08149718921446833, + "P4_HT_LD": 0.15670838558697175, + "P4_HT_PO": 0.9171870753137897, + "P4_LD": 0.05757759651927419, + "P4_ST_FD": 0.3068968241949937, + "P4_ST_GOV": 0.12759973358364962, + "P4_ST_LD": 0.15144925739836856, + "P4_ST_PO": 0.06906349752897034, + "P4_ST_PT01": 0.9214772329159236, + "P4_ST_TT01": 0.210431682939361 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.28779958871473243, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + }, + "avg_ks": 0.4130987421383648, + "avg_jsd": 0.017899328657243745, + "avg_lag1_diff": 0.3108961322076135 +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_quantile_loss__seed1337/eval_post.json b/example/results/ablations/runs/no_quantile_loss__seed1337/eval_post.json new file mode 100644 index 0000000..281155f --- /dev/null +++ b/example/results/ablations/runs/no_quantile_loss__seed1337/eval_post.json @@ -0,0 +1,971 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09355486979166669, + "std": 0.01799194293084728 + }, + "P1_B2016": { + "mean": 1.3915577083333348, + "std": 0.13947614022053412 + }, + "P1_B3004": { + "mean": 403.2905014583335, + "std": 14.217563527951576 + }, + "P1_B3005": { + "mean": 1059.5253898958326, + "std": 58.42587703751758 + }, + "P1_B4002": { + "mean": 32.37490617187499, + "std": 0.6519533213854061 + }, + "P1_B4005": { + "mean": 64.99571291666655, + "std": 42.03467331899641 + }, + "P1_B400B": { + "mean": 1772.6605813802094, + "std": 1195.5247571148332 + }, + "P1_B4022": { + "mean": 36.08671476562498, + "std": 0.7147782383668128 + }, + "P1_FCV02Z": { + "mean": 21.954832630208315, + "std": 38.87335152465215 + }, + "P1_FCV03D": { + "mean": 58.14541601562505, + "std": 6.267267768996455 + }, + "P1_FCV03Z": { + "mean": 58.21398343750003, + "std": 6.423211358701469 + }, + "P1_FT01": { + "mean": 194.93169489583332, + "std": 31.907421147789986 + }, + "P1_FT01Z": { + "mean": 877.3088108593755, + "std": 84.42922242684735 + }, + "P1_FT02": { + "mean": 1130.398466666666, + "std": 891.1246863991283 + }, + "P1_FT02Z": { + "mean": 1843.7261292187509, + "std": 1190.4241853713818 + }, + "P1_FT03": { + "mean": 282.4349720572918, + "std": 33.64962542238899 + }, + "P1_FT03Z": { + "mean": 1058.2564329947922, + "std": 59.50957255113764 + }, + "P1_LCV01D": { + "mean": 11.699055963541667, + "std": 3.0155193220433567 + }, + "P1_LCV01Z": { + "mean": 11.627510442708333, + "std": 3.0251072332459787 + }, + "P1_LIT01": { + "mean": 404.03647182291667, + "std": 15.824286512578475 + }, + "P1_PCV01D": { + "mean": 65.64593611979173, + "std": 23.56629129628467 + }, + "P1_PCV01Z": { + "mean": 65.565628671875, + "std": 23.790956943468952 + }, + "P1_PCV02Z": { + "mean": 11.867279765625007, + "std": 0.10811550492611105 + }, + "P1_PIT01": { + "mean": 1.366327500000001, + "std": 0.0673200977482619 + }, + "P1_PIT02": { + "mean": 0.22190658854166648, + "std": 0.05073903510741829 + }, + "P1_TIT01": { + "mean": 35.65866359374994, + "std": 0.5836675616735701 + }, + "P1_TIT02": { + "mean": 36.50122700520829, + "std": 1.2332879905119984 + }, + "P2_24Vdc": { + "mean": 28.02860809895833, + "std": 0.0032970984602375326 + }, + "P2_CO_rpm": { + "mean": 54102.283833828114, + "std": 24.087364527416046 + }, + "P2_HILout": { + "mean": 711.4986896354167, + "std": 7.856828438379631 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.2531249999992, + "std": 5.8591930957840725 + }, + "P2_SIT02": { + "mean": 783.923604296875, + "std": 5.842625008175601 + }, + "P2_VT01": { + "mean": 11.923782578124996, + "std": 0.06898207207491282 + }, + "P2_VXT02": { + "mean": -3.2992371614583327, + "std": 0.4773813056787038 + }, + "P2_VXT03": { + "mean": -1.3056961718749989, + "std": 0.3573689267646248 + }, + "P2_VYT02": { + "mean": 3.9808476822916674, + "std": 0.5056978789649539 + }, + "P2_VYT03": { + "mean": 6.233340729166663, + "std": 0.38920688744478077 + }, + "P3_FIT01": { + "mean": 888.4507813802076, + "std": 1676.1943844854939 + }, + "P3_LCP01D": { + "mean": 4590.880697031249, + "std": 5143.4225118731565 + }, + "P3_LCV01D": { + "mean": 7134.490903958336, + "std": 6860.174505041899 + }, + "P3_LIT01": { + "mean": 13352.644031640628, + "std": 4370.946869247855 + }, + "P3_PIT01": { + "mean": 392.9633114062501, + "std": 988.8695443670978 + }, + "P4_HT_FD": { + "mean": -0.0004982812500000002, + "std": 0.0044673488687889595 + }, + "P4_HT_LD": { + "mean": 34.231568880208286, + "std": 33.60361854009684 + }, + "P4_HT_PO": { + "mean": 35.25451283854166, + "std": 31.650898099038645 + }, + "P4_LD": { + "mean": 366.12485794270833, + "std": 61.02942352172124 + }, + "P4_ST_FD": { + "mean": -6.640625000000009e-05, + "std": 0.004044489801718016 + }, + "P4_ST_GOV": { + "mean": 17918.738952656262, + "std": 1897.4704889747775 + }, + "P4_ST_LD": { + "mean": 332.09368789062484, + "std": 38.593957056472306 + }, + "P4_ST_PO": { + "mean": 332.36840252604156, + "std": 35.260567478365545 + }, + "P4_ST_PT01": { + "mean": 10045.100718177076, + "std": 19.20451867005973 + }, + "P4_ST_TT01": { + "mean": 27608.458333333358, + "std": 24.483240568921808 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 1.2782299308983225e-05, + "std_abs_err": 6.494151245123314e-06 + }, + "P1_B2016": { + "mean_abs_err": 0.00029138366856784614, + "std_abs_err": 0.003702577743971036 + }, + "P1_B3004": { + "mean_abs_err": 0.09556236619266656, + "std_abs_err": 1.2005860600902345 + }, + "P1_B3005": { + "mean_abs_err": 1.5233966327971302, + "std_abs_err": 2.757439473482698 + }, + "P1_B4002": { + "mean_abs_err": 0.05797985199510691, + "std_abs_err": 0.07327982218632823 + }, + "P1_B4005": { + "mean_abs_err": 4.470605900468627, + "std_abs_err": 0.14662412871327746 + }, + "P1_B400B": { + "mean_abs_err": 5.81528879775874, + "std_abs_err": 2.1724841957604895 + }, + "P1_B4022": { + "mean_abs_err": 0.02554011822245883, + "std_abs_err": 0.04332738353199439 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.996102270263961, + "std_abs_err": 1.5651339146060295 + }, + "P1_FCV03D": { + "mean_abs_err": 0.42244743522177686, + "std_abs_err": 0.3322279497122995 + }, + "P1_FCV03Z": { + "mean_abs_err": 1.0850557426898035, + "std_abs_err": 0.29159260563013323 + }, + "P1_FT01": { + "mean_abs_err": 0.20804053955305335, + "std_abs_err": 4.5594393091456595 + }, + "P1_FT01Z": { + "mean_abs_err": 1.7408706447770328, + "std_abs_err": 8.663519463529099 + }, + "P1_FT02": { + "mean_abs_err": 2.4196865679600705, + "std_abs_err": 1.9285225477157155 + }, + "P1_FT02Z": { + "mean_abs_err": 65.24768261010217, + "std_abs_err": 2.9211703997677887 + }, + "P1_FT03": { + "mean_abs_err": 0.20399821893710168, + "std_abs_err": 0.17541231109348843 + }, + "P1_FT03Z": { + "mean_abs_err": 2.785605353729352, + "std_abs_err": 1.6159163124796905 + }, + "P1_LCV01D": { + "mean_abs_err": 0.45527669823346884, + "std_abs_err": 0.1009043690132354 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.33856358662958463, + "std_abs_err": 0.061637735900667234 + }, + "P1_LIT01": { + "mean_abs_err": 0.07840034008961538, + "std_abs_err": 0.1923884642045035 + }, + "P1_PCV01D": { + "mean_abs_err": 1.8196376158996515, + "std_abs_err": 0.20347540462090308 + }, + "P1_PCV01Z": { + "mean_abs_err": 2.842523125027583, + "std_abs_err": 0.07364286719469249 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.024526819020080026, + "std_abs_err": 0.011271039461463683 + }, + "P1_PIT01": { + "mean_abs_err": 0.011153720753390939, + "std_abs_err": 0.005887538378379804 + }, + "P1_PIT02": { + "mean_abs_err": 0.2652505017998502, + "std_abs_err": 0.44797260294746155 + }, + "P1_TIT01": { + "mean_abs_err": 0.022544648683272328, + "std_abs_err": 0.0003869344876211711 + }, + "P1_TIT02": { + "mean_abs_err": 0.023751673128238338, + "std_abs_err": 0.011529264487735391 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00010160366933931186, + "std_abs_err": 4.536765416938244e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.426364054561418, + "std_abs_err": 1.6316327526276808 + }, + "P2_HILout": { + "mean_abs_err": 0.035880365320508645, + "std_abs_err": 0.5978974388765312 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.6507924134438099, + "std_abs_err": 0.6602777353398475 + }, + "P2_SIT02": { + "mean_abs_err": 0.019866414328816973, + "std_abs_err": 0.652045251362634 + }, + "P2_VT01": { + "mean_abs_err": 3.3067885137683106e-05, + "std_abs_err": 0.00046839000291075406 + }, + "P2_VXT02": { + "mean_abs_err": 0.0009413109316636081, + "std_abs_err": 0.0063483794074835065 + }, + "P2_VXT03": { + "mean_abs_err": 0.001197114789678988, + "std_abs_err": 0.0077415476009711215 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006211677012695205, + "std_abs_err": 0.005787425972166027 + }, + "P2_VYT03": { + "mean_abs_err": 0.0010643836312214106, + "std_abs_err": 0.009403925473689179 + }, + "P3_FIT01": { + "mean_abs_err": 262.4493947259756, + "std_abs_err": 103.53031330478098 + }, + "P3_LCP01D": { + "mean_abs_err": 6.153548790674904, + "std_abs_err": 13.422514157277874 + }, + "P3_LCV01D": { + "mean_abs_err": 0.614186977094505, + "std_abs_err": 20.246609226077453 + }, + "P3_LIT01": { + "mean_abs_err": 2.2004610533585947, + "std_abs_err": 16.772232319918658 + }, + "P3_PIT01": { + "mean_abs_err": 256.3945916170776, + "std_abs_err": 162.05238634009754 + }, + "P4_HT_FD": { + "mean_abs_err": 0.00039509330464826046, + "std_abs_err": 0.002435330554384859 + }, + "P4_HT_LD": { + "mean_abs_err": 0.23665180052267232, + "std_abs_err": 0.17982503009853446 + }, + "P4_HT_PO": { + "mean_abs_err": 1.2707050437847514, + "std_abs_err": 0.21002688171716954 + }, + "P4_LD": { + "mean_abs_err": 0.0038958514859928073, + "std_abs_err": 0.4136171522782348 + }, + "P4_ST_FD": { + "mean_abs_err": 3.729302011685127e-05, + "std_abs_err": 0.002402313543727987 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.777691408737155, + "std_abs_err": 45.32727298700411 + }, + "P4_ST_LD": { + "mean_abs_err": 0.09079072080714923, + "std_abs_err": 0.7407877648547583 + }, + "P4_ST_PO": { + "mean_abs_err": 0.0994312023827888, + "std_abs_err": 0.8096239759572441 + }, + "P4_ST_PT01": { + "mean_abs_err": 5.474438642006135, + "std_abs_err": 1.4197486447406007 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.5743583461735398, + "std_abs_err": 0.56591876435348 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.6595583333333334, + "P1_B2016": 0.043591666666666584, + "P1_B3004": 0.671875, + "P1_B3005": 0.5293, + "P1_B4002": 0.7923208333333334, + "P1_B4005": 0.3062625, + "P1_B400B": 0.46663333333333334, + "P1_B4022": 0.5756333333333334, + "P1_FCV02Z": 0.7746, + "P1_FCV03D": 0.45162500000000005, + "P1_FCV03Z": 0.40985833333333327, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4746, + "P1_FT02": 0.4630875, + "P1_FT02Z": 0.4029958333333334, + "P1_FT03": 0.5241, + "P1_FT03Z": 0.46950833333333336, + "P1_LCV01D": 0.3859916666666667, + "P1_LCV01Z": 0.397625, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5304583333333333, + "P1_PCV01Z": 0.5184416666666667, + "P1_PCV02Z": 0.6588541666666666, + "P1_PIT01": 0.15849999999999997, + "P1_PIT02": 0.61435, + "P1_TIT01": 0.10056666666666658, + "P1_TIT02": 0.10090416666666668, + "P2_24Vdc": 0.13548333333333334, + "P2_CO_rpm": 0.09535833333333332, + "P2_HILout": 0.08242499999999997, + "P2_MSD": 0.0, + "P2_SIT01": 0.4783083333333333, + "P2_SIT02": 0.5016666666666667, + "P2_VT01": 0.12292083333333337, + "P2_VXT02": 0.2814458333333334, + "P2_VXT03": 0.43662083333333335, + "P2_VYT02": 0.250625, + "P2_VYT03": 0.26955416666666665, + "P3_FIT01": 0.22947499999999998, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.07123333333333334, + "P3_LIT01": 0.09349166666666664, + "P3_PIT01": 0.6393375, + "P4_HT_FD": 0.148725, + "P4_HT_LD": 0.06742500000000001, + "P4_HT_PO": 0.05970416666666667, + "P4_LD": 0.031137499999999985, + "P4_ST_FD": 0.028475000000000028, + "P4_ST_GOV": 0.03517083333333337, + "P4_ST_LD": 0.03240416666666668, + "P4_ST_PO": 0.035720833333333424, + "P4_ST_PT01": 0.38136666666666663, + "P4_ST_TT01": 0.06839166666666663 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.02845, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0011099999999999999, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.011949999999999905, + "q25_diff": 0.010229999999999961, + "q50_diff": 0.005139999999999922, + "q75_diff": 0.014319999999999888, + "q95_diff": 0.029889999999999972 + }, + "P1_B3004": { + "q05_diff": 14.80313000000001, + "q25_diff": 1.6197799999999916, + "q50_diff": 6.726189999999974, + "q75_diff": 8.508819999999957, + "q95_diff": 37.86444 + }, + "P1_B3005": { + "q05_diff": 40.40789999999993, + "q25_diff": 8.045049999999947, + "q50_diff": 89.51202, + "q75_diff": 104.97393, + "q95_diff": 4.4593499999998585 + }, + "P1_B4002": { + "q05_diff": 0.4564699999999995, + "q25_diff": 1.6555000000000035, + "q50_diff": 1.6555000000000035, + "q75_diff": 0.9287000000000063, + "q95_diff": 0.16969000000000278 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 81.96193, + "q50_diff": 0.04000000000000625, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2479.8766800000003, + "q50_diff": 284.0803299999998, + "q75_diff": 9.545890000000327, + "q95_diff": 7.159419999999955 + }, + "P1_B4022": { + "q05_diff": 0.38222999999999985, + "q25_diff": 1.3479000000000028, + "q50_diff": 1.4273799999999994, + "q75_diff": 0.9127399999999994, + "q95_diff": 0.5270299999999963 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.05340999999999996, + "q75_diff": 38.169869999999996, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 1.3654200000000003, + "q25_diff": 0.2554099999999977, + "q50_diff": 8.60333, + "q75_diff": 10.84863, + "q95_diff": 0.824300000000008 + }, + "P1_FCV03Z": { + "q05_diff": 1.7547599999999974, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.148199999999996, + "q75_diff": 10.864249999999998, + "q95_diff": 0.9307899999999876 + }, + "P1_FT01": { + "q05_diff": 8.088040000000007, + "q25_diff": 8.587160000000011, + "q50_diff": 32.806329999999974, + "q75_diff": 37.95618999999999, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 38.51202999999998, + "q25_diff": 19.571230000000014, + "q50_diff": 74.79840000000002, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1898.00245, + "q50_diff": 258.24926000000005, + "q75_diff": 12.970209999999952, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 1.9862300000000026, + "q25_diff": 2300.54303, + "q50_diff": 14.318849999999657, + "q75_diff": 7.299800000000232, + "q95_diff": 6.317140000000109 + }, + "P1_FT03": { + "q05_diff": 35.89179999999999, + "q25_diff": 1.5264000000000237, + "q50_diff": 51.52633000000003, + "q75_diff": 59.766309999999976, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 23.41980000000001, + "q25_diff": 2.9805299999999306, + "q50_diff": 84.98266000000012, + "q75_diff": 100.6383699999999, + "q95_diff": 3.5079399999999623 + }, + "P1_LCV01D": { + "q05_diff": 0.17497000000000007, + "q25_diff": 0.29953000000000074, + "q50_diff": 4.171619999999999, + "q75_diff": 3.162939999999999, + "q95_diff": 0.7125800000000009 + }, + "P1_LCV01Z": { + "q05_diff": 0.0, + "q25_diff": 0.5874600000000001, + "q50_diff": 4.638670000000001, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.9536700000000007 + }, + "P1_LIT01": { + "q05_diff": 14.39278999999999, + "q25_diff": 0.6645500000000197, + "q50_diff": 6.283670000000029, + "q75_diff": 8.229829999999993, + "q95_diff": 34.78140000000002 + }, + "P1_PCV01D": { + "q05_diff": 0.18625000000000114, + "q25_diff": 2.7490099999999984, + "q50_diff": 18.274499999999996, + "q75_diff": 43.266600000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.22888000000000375, + "q25_diff": 2.0370500000000007, + "q50_diff": 16.029360000000004, + "q75_diff": 43.33495, + "q95_diff": 45.1493 + }, + "P1_PCV02Z": { + "q05_diff": 0.2288800000000002, + "q25_diff": 0.2212499999999995, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_PIT01": { + "q05_diff": 0.01373000000000002, + "q25_diff": 0.008400000000000185, + "q50_diff": 0.010680000000000023, + "q75_diff": 0.02364999999999995, + "q95_diff": 0.008389999999999898 + }, + "P1_PIT02": { + "q05_diff": 0.03051999999999999, + "q25_diff": 0.10146999999999998, + "q50_diff": 0.10223999999999997, + "q75_diff": 0.0015300000000000313, + "q95_diff": 0.0022900000000000142 + }, + "P1_TIT01": { + "q05_diff": 0.015250000000001762, + "q25_diff": 0.06103999999999843, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.24414999999999765, + "q95_diff": 0.061029999999995255 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.030520000000002767, + "q50_diff": 0.09154999999999802, + "q75_diff": 0.30516999999999683, + "q95_diff": 0.4272400000000047 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.00013999999999825263, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.00010999999999938836 + }, + "P2_CO_rpm": { + "q05_diff": 10.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 3.0, + "q95_diff": 0.0 + }, + "P2_HILout": { + "q05_diff": 0.3295900000000529, + "q25_diff": 1.6540499999999838, + "q50_diff": 0.31737999999995736, + "q75_diff": 1.7334000000000742, + "q95_diff": 2.2399900000000343 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 1.0, + "q25_diff": 3.0, + "q50_diff": 5.0, + "q75_diff": 6.0, + "q95_diff": 6.0 + }, + "P2_SIT02": { + "q05_diff": 2.6579000000000406, + "q25_diff": 3.569340000000011, + "q50_diff": 5.617490000000089, + "q75_diff": 6.36414000000002, + "q95_diff": 6.8431399999999485 + }, + "P2_VT01": { + "q05_diff": 0.008470000000000866, + "q25_diff": 0.015229999999998967, + "q50_diff": 0.014129999999999754, + "q75_diff": 0.013349999999999085, + "q95_diff": 0.01886999999999972 + }, + "P2_VXT02": { + "q05_diff": 0.1316500000000005, + "q25_diff": 0.2994399999999997, + "q50_diff": 0.28542999999999985, + "q75_diff": 0.26605000000000034, + "q95_diff": 0.4595699999999998 + }, + "P2_VXT03": { + "q05_diff": 0.10078999999999994, + "q25_diff": 0.26920999999999995, + "q50_diff": 0.29757999999999996, + "q75_diff": 0.32704, + "q95_diff": 0.61108 + }, + "P2_VYT02": { + "q05_diff": 0.0950899999999999, + "q25_diff": 0.26207, + "q50_diff": 0.2269300000000003, + "q75_diff": 0.23554999999999993, + "q95_diff": 0.3890600000000006 + }, + "P2_VYT03": { + "q05_diff": 0.05600999999999967, + "q25_diff": 0.18581000000000003, + "q50_diff": 0.1761499999999998, + "q75_diff": 0.18344999999999967, + "q95_diff": 0.3527800000000001 + }, + "P3_FIT01": { + "q05_diff": 3.0, + "q25_diff": 2.0, + "q50_diff": 72.0, + "q75_diff": 2484.0, + "q95_diff": 18.942869999999857 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 101.40417000000002, + "q75_diff": 135.01952999999958, + "q95_diff": 8.21484000000055 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 256.0, + "q50_diff": 3226.9873, + "q75_diff": 174.61621000000014, + "q95_diff": 31.333010000000286 + }, + "P3_LIT01": { + "q05_diff": 1132.9770500000004, + "q25_diff": 34.37890999999945, + "q50_diff": 647.4140599999992, + "q75_diff": 1151.4589799999994, + "q95_diff": 1.6015599999991537 + }, + "P3_PIT01": { + "q05_diff": 1.0, + "q25_diff": 2.0, + "q50_diff": 3.0, + "q75_diff": 508.0, + "q95_diff": 188.0 + }, + "P4_HT_FD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 7e-05, + "q95_diff": 0.00021999999999999993 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 27.14122, + "q75_diff": 0.13020999999999106, + "q95_diff": 0.21701000000000192 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 0.9223299999999999, + "q50_diff": 8.89752, + "q75_diff": 0.054199999999994475, + "q95_diff": 0.6510700000000043 + }, + "P4_LD": { + "q05_diff": 0.5004900000000134, + "q25_diff": 3.1075700000000097, + "q50_diff": 3.4702399999999898, + "q75_diff": 0.07668999999998505, + "q95_diff": 4.30313000000001 + }, + "P4_ST_FD": { + "q05_diff": 6.999999999999975e-05, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 8.000000000000021e-05 + }, + "P4_ST_GOV": { + "q05_diff": 116.37305000000015, + "q25_diff": 38.04102000000057, + "q50_diff": 29.617190000000846, + "q75_diff": 109.16602000000057, + "q95_diff": 404.4824200000003 + }, + "P4_ST_LD": { + "q05_diff": 0.5155600000000504, + "q25_diff": 0.5112599999999929, + "q50_diff": 0.3724400000000401, + "q75_diff": 2.00551999999999, + "q95_diff": 7.728729999999985 + }, + "P4_ST_PO": { + "q05_diff": 2.1687599999999634, + "q25_diff": 0.8120700000000056, + "q50_diff": 0.5768199999999979, + "q75_diff": 2.1424200000000155, + "q95_diff": 7.583590000000015 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 20.0, + "q50_diff": 1.0, + "q75_diff": 1.0, + "q95_diff": 30.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 7.0, + "q50_diff": 3.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0624831607922858, + "P1_B2016": 0.07618695239144935, + "P1_B3004": 0.9420514424699565, + "P1_B3005": 0.9755864990985265, + "P1_B4002": 0.9939428775878422, + "P1_B4005": 1.0901635813805555, + "P1_B400B": 0.45045912562159773, + "P1_B4022": 1.0728287547128275, + "P1_FCV02Z": 1.0856923346443097, + "P1_FCV03D": 1.0287616800381814, + "P1_FCV03Z": 1.01920928123416, + "P1_FT01": 0.5820973686096496, + "P1_FT01Z": 0.3340826131634087, + "P1_FT02": 0.75191076012623, + "P1_FT02Z": 1.0572315062387077, + "P1_FT03": 0.2834155903958254, + "P1_FT03Z": 0.9861265787637634, + "P1_LCV01D": 0.9922897795117445, + "P1_LCV01Z": 0.9035160861736835, + "P1_LIT01": 0.12178882881233055, + "P1_PCV01D": 0.9847169107083952, + "P1_PCV01Z": 1.1483081333334428, + "P1_PCV02Z": 0.5764046458797007, + "P1_PIT01": 0.22468855469933502, + "P1_PIT02": 0.06124155098334361, + "P1_TIT01": 0.04468432299948122, + "P1_TIT02": 0.06696073912814682, + "P2_24Vdc": 0.039965792976989475, + "P2_CO_rpm": 0.34346874863740007, + "P2_HILout": 0.24706059027478758, + "P2_MSD": 0.0, + "P2_SIT01": 0.3875131407531049, + "P2_SIT02": 0.4052675740532808, + "P2_VT01": 0.14773170910846045, + "P2_VXT02": 0.163033385889365, + "P2_VXT03": 0.18198369820033933, + "P2_VYT02": 0.1452756215000487, + "P2_VYT03": 0.16882683150174038, + "P3_FIT01": 0.2716177197527262, + "P3_LCP01D": 0.5618088756774389, + "P3_LCV01D": 0.43449899105168155, + "P3_LIT01": 0.07644556223581167, + "P3_PIT01": 0.8737468921804842, + "P4_HT_FD": 0.08149718921446833, + "P4_HT_LD": 1.052071148480924, + "P4_HT_PO": 1.0918064404320895, + "P4_LD": 0.05757759651927419, + "P4_ST_FD": 0.3068968241949937, + "P4_ST_GOV": 0.12759973358364962, + "P4_ST_LD": 0.15144925739836856, + "P4_ST_PO": 0.06906349752897034, + "P4_ST_PT01": 0.9214772329159236, + "P4_ST_TT01": 0.210431682939361 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.28779958871473243, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_residual_stat__seed1337/comprehensive_eval.json b/example/results/ablations/runs/no_residual_stat__seed1337/comprehensive_eval.json new file mode 100644 index 0000000..c2e9397 --- /dev/null +++ b/example/results/ablations/runs/no_residual_stat__seed1337/comprehensive_eval.json @@ -0,0 +1,537 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_residual_stat__seed1337/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.4039510220125786, + "avg_jsd": 0.02930392024585336, + "avg_lag1_diff": 0.28531566380969187 + }, + "two_sample": { + "continuous_mmd_rbf": 0.6561890840530396, + "continuous_mmd_gamma": 1.7111706960948813e-05, + "histogram_mmd_rbf": 0.6561899185180664, + "histogram_mmd_gamma": 1.711084001123968e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 162.96432495117188, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 2.953516468815496, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.21048887073993683, + "corr_frobenius": 16.381309509277344, + "lag1_corr_mean_abs_diff": 0.21117864549160004, + "lag1_corr_frobenius": 16.544153213500977, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.29415977001190186, + "corr_frobenius": 10.300400733947754, + "lag1_corr_mean_abs_diff": 0.30442026257514954, + "lag1_corr_frobenius": 10.519073486328125 + }, + "P2": { + "corr_mean_abs_diff": 0.08638041466474533, + "corr_frobenius": 1.9565529823303223, + "lag1_corr_mean_abs_diff": 0.15203315019607544, + "lag1_corr_frobenius": 2.7942450046539307 + }, + "P3": { + "corr_mean_abs_diff": 0.4491882622241974, + "corr_frobenius": 3.076937437057495, + "lag1_corr_mean_abs_diff": 0.4927428960800171, + "lag1_corr_frobenius": 3.1510467529296875 + }, + "P4": { + "corr_mean_abs_diff": 0.18113404512405396, + "corr_frobenius": 2.8871641159057617, + "lag1_corr_mean_abs_diff": 0.19903287291526794, + "lag1_corr_frobenius": 3.0042483806610107 + } + } + }, + "frequency": { + "avg_psd_l1": 0.019299142062664032, + "avg_psd_cosine": 0.33642748325220434, + "avg_low_high_ratio_abs_diff": 9433974.760899194 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 15.0, + "mean_dwell": 24.0, + "median_dwell": 2.5, + "mean_step": 68.42660395304362, + "median_step": 1.9043693542480469 + }, + "P1_B2004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B3004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.01115664489586733, + "median_step": 0.004810333251953125 + }, + "P1_B3005": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22998.14285714286, + "mean_abs_diff_mean_dwell": 14201.625685203559, + "mean_abs_diff_median_dwell": 14207.642857142857, + "mean_abs_diff_mean_step": 33.693117622162895, + "mean_abs_diff_median_step": 0.831573486328125 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 1.0, + "change_rate": 0.0, + "step_median": NaN + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.411041259765625, + "mean_abs_diff_change_rate": 0.4608201173921447 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.0026041666666666665, + "top1_mass": 1.0, + "top3_mass": 1.0, + "median_dwell": 384.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8984375, + "top1_mass": 0.020833333333333332, + "top3_mass": 0.052083333333333336, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9192708333333334, + "top1_mass": 0.08333333333333333, + "top3_mass": 0.08854166666666667, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.036458333333333336, + "top1_mass": 0.3177083333333333, + "top3_mass": 0.6197916666666666, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.9973958333333334, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.010416666666666666, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.390625, + "top1_mass": 0.11197916666666667, + "top3_mass": 0.21875, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.9713541666666666, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.015625, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.21354166666666666, + "top3_mass": 0.40625, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.528283437093099, + "mean_abs_diff_top1_mass": 0.10245386759440103, + "mean_abs_diff_top3_mass": 0.08072535196940105, + "mean_abs_diff_median_dwell": 47.75 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17486999928951263, + "q50": 0.27557000517845154, + "q95": 0.27557000517845154, + "tail_ratio": 0.0 + }, + "P2_SIT02": { + "q05": 774.9249877929688, + "q50": 784.5618286132812, + "q95": 791.7132568359375, + "tail_ratio": 0.7420925961112167 + }, + "P1_FT03": { + "q05": 209.25038146972656, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.19322954821396512 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.687569265564283, + "mean_abs_diff_q50": 2.23627961675326, + "mean_abs_diff_q95": 3.5624012251695, + "mean_abs_diff_tail_ratio": 7.229591227292186 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 379.0, + "mean_dwell": 1.0105263157894737, + "median_dwell": 1.0, + "mean_step": 18.72084262817705, + "median_step": 7.819976806640625 + }, + "P1_FT02Z": { + "num_changes": 377.0, + "mean_dwell": 1.0158730158730158, + "median_dwell": 1.0, + "mean_step": 687.4984294248828, + "median_step": 175.9818115234375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89284.5, + "mean_abs_diff_mean_dwell": 0.08773083450101415, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 345.45853545101454, + "mean_abs_diff_median_step": 88.63398742675781 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 33.99760055541992, + "std": 31.544307708740234, + "lag1": 0.9009254574775696 + }, + "P2_24Vdc": { + "mean": 28.028615951538086, + "std": 0.0035392381250858307, + "lag1": 0.16610850393772125 + }, + "P2_HILout": { + "mean": 711.4860229492188, + "std": 9.420388221740723, + "lag1": -0.031849831342697144 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.28293482462565106, + "mean_abs_diff_std": 0.46856609522365034, + "mean_abs_diff_lag1": 0.16597836837172508 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9696047902107239, + "mae": 0.778528094291687 + }, + "real_plus_synthetic": { + "rmse": 0.5507087111473083, + "mae": 0.3574029207229614 + }, + "rmse_ratio_synth_to_real": 1.7365508935397789 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8305658627087199, + "auprc": 0.5848963274786471, + "threshold": 0.005620298907160759, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6756756756756758, + "best_f1_threshold": 1.4679230451583862 + }, + "real_plus_synthetic": { + "auroc": 0.8808209647495361, + "auprc": 0.6465074146607899, + "threshold": 0.3127894401550293, + "f1": 0.24999999999999997, + "precision": 0.15196078431372548, + "recall": 0.7045454545454546, + "best_f1": 0.7532467532467532, + "best_f1_threshold": 0.4812386929988861 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_residual_stat__seed1337/comprehensive_eval_post.json b/example/results/ablations/runs/no_residual_stat__seed1337/comprehensive_eval_post.json new file mode 100644 index 0000000..85ceab5 --- /dev/null +++ b/example/results/ablations/runs/no_residual_stat__seed1337/comprehensive_eval_post.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_residual_stat__seed1337/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.21142470836639404, + "continuous_mmd_gamma": 5.127790138778833e-05, + "histogram_mmd_rbf": 0.21151018142700195, + "histogram_mmd_gamma": 5.127251471756505e-05, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 90.39964294433594, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 1.6383759714964339, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.18907257914543152, + "corr_frobenius": 15.711627960205078, + "lag1_corr_mean_abs_diff": 0.19814208149909973, + "lag1_corr_frobenius": 16.458852767944336, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.33941954374313354, + "corr_frobenius": 11.760598182678223, + "lag1_corr_mean_abs_diff": 0.36233869194984436, + "lag1_corr_frobenius": 12.461121559143066 + }, + "P2": { + "corr_mean_abs_diff": 0.06258689612150192, + "corr_frobenius": 1.1153255701065063, + "lag1_corr_mean_abs_diff": 0.1492249220609665, + "lag1_corr_frobenius": 2.822643995285034 + }, + "P3": { + "corr_mean_abs_diff": 0.4491882622241974, + "corr_frobenius": 3.076937437057495, + "lag1_corr_mean_abs_diff": 0.4927428960800171, + "lag1_corr_frobenius": 3.1510467529296875 + }, + "P4": { + "corr_mean_abs_diff": 0.19689571857452393, + "corr_frobenius": 3.1779367923736572, + "lag1_corr_mean_abs_diff": 0.22919553518295288, + "lag1_corr_frobenius": 3.538879156112671 + } + } + }, + "frequency": { + "avg_psd_l1": 0.022606568410992622, + "avg_psd_cosine": 0.44756376181008684, + "avg_low_high_ratio_abs_diff": 18.2513081884368 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 308.0, + "mean_dwell": 1.2427184466019416, + "median_dwell": 1.0, + "mean_step": 0.8413416007896523, + "median_step": 0.7274894714355469 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 347.0, + "mean_dwell": 1.103448275862069, + "median_dwell": 1.0, + "mean_step": 41.3141472878254, + "median_step": 54.116028785705566 + }, + "P1_B2004": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 0.014068839402653235, + "median_step": 0.0017450004816055298 + }, + "P1_B3004": { + "num_changes": 370.0, + "mean_dwell": 1.0350404312668464, + "median_dwell": 1.0, + "mean_step": 14.77256890374261, + "median_step": 10.804779052734375 + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.8157603410765646, + "median_step": 0.5944099426269531 + }, + "P1_B3005": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 64.28072175926926, + "median_step": 41.228179931640625 + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22997.85714285714, + "mean_abs_diff_mean_dwell": 14417.281142645239, + "mean_abs_diff_median_dwell": 13988.57142857143, + "mean_abs_diff_mean_step": 19.03385039702909, + "mean_abs_diff_median_step": 16.840154950817425 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6822916666666666, + "change_rate": 0.7702349869451697, + "step_median": 68.81373023986816 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.09333292643229163, + "mean_abs_diff_change_rate": 0.309414869553025, + "mean_abs_diff_step_median": 68.78170993924141 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.015625, + "top1_mass": 0.40625, + "top3_mass": 0.7760416666666666, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8776041666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.059895833333333336, + "top3_mass": 0.0703125, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.18229166666666666, + "top3_mass": 0.3697916666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.8463541666666666, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4817708333333333, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.7864583333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3515625, + "top1_mass": 0.036458333333333336, + "top3_mass": 0.09635416666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5303484598795573, + "mean_abs_diff_top1_mass": 0.03112411499023438, + "mean_abs_diff_top3_mass": 0.05629857381184896, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17486999928951263, + "q50": 0.27557000517845154, + "q95": 0.27557000517845154, + "tail_ratio": 0.0 + }, + "P2_SIT02": { + "q05": 774.9249877929688, + "q50": 784.5618286132812, + "q95": 791.7132568359375, + "tail_ratio": 0.7420925961112167 + }, + "P1_FT03": { + "q05": 209.25038146972656, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.19322954821396512 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.687569265564283, + "mean_abs_diff_q50": 2.23627961675326, + "mean_abs_diff_q95": 3.5624012251695, + "mean_abs_diff_tail_ratio": 7.229591227292186 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 64.74391126130756, + "median_step": 62.410614013671875 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 1312.3134692358592, + "median_step": 1284.6439208984375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.09039718638964156, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 680.877589673068, + "mean_abs_diff_median_step": 670.2603607177734 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 35.254512786865234, + "std": 31.650897979736328, + "lag1": -0.0927804708480835 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.003297113813459873, + "lag1": 0.06643600016832352 + }, + "P2_HILout": { + "mean": 711.4987182617188, + "std": 7.856828212738037, + "lag1": -0.013758387416601181 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.4565467834472656, + "mean_abs_diff_std": 0.2373291968833655, + "mean_abs_diff_lag1": 0.4579590285817782 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9678412675857544, + "mae": 0.7817822694778442 + }, + "real_plus_synthetic": { + "rmse": 0.5488210320472717, + "mae": 0.35390278697013855 + }, + "rmse_ratio_synth_to_real": 1.7333924450450031 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8277597402597403, + "auprc": 0.5840426125640282, + "threshold": 0.00809176079928875, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6756756756756758, + "best_f1_threshold": 1.468359351158142 + }, + "real_plus_synthetic": { + "auroc": 0.876252319109462, + "auprc": 0.6427478864536372, + "threshold": 0.28309258818626404, + "f1": 0.24150943396226415, + "precision": 0.14479638009049775, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.5237401127815247 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_residual_stat__seed1337/eval.json b/example/results/ablations/runs/no_residual_stat__seed1337/eval.json new file mode 100644 index 0000000..18175fa --- /dev/null +++ b/example/results/ablations/runs/no_residual_stat__seed1337/eval.json @@ -0,0 +1,974 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.11384, + "std": 0.0 + }, + "P1_B2016": { + "mean": 1.391552838541667, + "std": 0.13945898473945204 + }, + "P1_B3004": { + "mean": 396.00211, + "std": 0.0 + }, + "P1_B3005": { + "mean": 1015.43976, + "std": 0.0 + }, + "P1_B4002": { + "mean": 33.54375, + "std": 0.0 + }, + "P1_B4005": { + "mean": 0.0, + "std": 0.0 + }, + "P1_B400B": { + "mean": 1774.5945444791669, + "std": 1195.7479847272803 + }, + "P1_B4022": { + "mean": 36.881583541666664, + "std": 0.14105102171131376 + }, + "P1_FCV02Z": { + "mean": 22.690971406250004, + "std": 40.331634476922204 + }, + "P1_FCV03D": { + "mean": 58.5711173958333, + "std": 6.6408459010316205 + }, + "P1_FCV03Z": { + "mean": 59.04171070312495, + "std": 6.889645189398308 + }, + "P1_FT01": { + "mean": 194.8422163802083, + "std": 32.042278691765326 + }, + "P1_FT01Z": { + "mean": 877.036905338541, + "std": 84.60979134730329 + }, + "P1_FT02": { + "mean": 1127.59378625, + "std": 889.9987774385405 + }, + "P1_FT02Z": { + "mean": 1127.59378625, + "std": 889.9987774385405 + }, + "P1_FT03": { + "mean": 282.5606103906248, + "std": 33.7615481029071 + }, + "P1_FT03Z": { + "mean": 282.5606103906248, + "std": 33.7615481029071 + }, + "P1_LCV01D": { + "mean": 12.167560755208338, + "std": 3.0459293388132886 + }, + "P1_LCV01Z": { + "mean": 11.602465937499993, + "std": 3.093944140157223 + }, + "P1_LIT01": { + "mean": 404.03977184895825, + "std": 15.869543048655954 + }, + "P1_PCV01D": { + "mean": 67.44972924479171, + "std": 23.85317503843504 + }, + "P1_PCV01Z": { + "mean": 68.33526786458337, + "std": 23.953987663086707 + }, + "P1_PCV02Z": { + "mean": 11.76605, + "std": 0.0 + }, + "P1_PIT01": { + "mean": 1.369015130208333, + "std": 0.08488627090083745 + }, + "P1_PIT02": { + "mean": 0.22518221354166668, + "std": 0.05045513920933258 + }, + "P1_TIT01": { + "mean": 35.66156757812503, + "std": 0.584167110393421 + }, + "P1_TIT02": { + "mean": 36.50086203124998, + "std": 1.238351777565239 + }, + "P2_24Vdc": { + "mean": 28.028612578125006, + "std": 0.003539207751496049 + }, + "P2_CO_rpm": { + "mean": 54102.19458007815, + "std": 24.034860936309418 + }, + "P2_HILout": { + "mean": 711.4860583072921, + "std": 9.420388671382987 + }, + "P2_MSD": { + "mean": 758.4425, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.0494791666662, + "std": 5.881346335233885 + }, + "P2_SIT02": { + "mean": 783.9242636718743, + "std": 5.840092885735959 + }, + "P2_VT01": { + "mean": 11.92377749999999, + "std": 0.06898678712937246 + }, + "P2_VXT02": { + "mean": -3.2993033072916664, + "std": 0.4773571042493072 + }, + "P2_VXT03": { + "mean": -1.3056782552083335, + "std": 0.35732155885605127 + }, + "P2_VYT02": { + "mean": 3.9808272135416694, + "std": 0.5059750232396318 + }, + "P2_VYT03": { + "mean": 6.233439661458335, + "std": 0.3891382633670636 + }, + "P3_FIT01": { + "mean": 985.2521917708335, + "std": 1710.6766945035433 + }, + "P3_LCP01D": { + "mean": 4590.872553229171, + "std": 5143.480127672877 + }, + "P3_LCV01D": { + "mean": 7094.691859296871, + "std": 6873.739921920712 + }, + "P3_LIT01": { + "mean": 13352.421090390619, + "std": 4371.226979602155 + }, + "P3_PIT01": { + "mean": 119.68190002604165, + "std": 696.7225876768903 + }, + "P4_HT_FD": { + "mean": -0.0004619791666666666, + "std": 0.004412049338174849 + }, + "P4_HT_LD": { + "mean": -115.69799739583333, + "std": 35.27327640165136 + }, + "P4_HT_PO": { + "mean": 33.99760059895832, + "std": 31.544307343476465 + }, + "P4_LD": { + "mean": 366.12431692708344, + "std": 61.03208266436402 + }, + "P4_ST_FD": { + "mean": -6.843750000000004e-05, + "std": 0.004047121938568158 + }, + "P4_ST_GOV": { + "mean": 17918.64454398437, + "std": 1897.1499748837282 + }, + "P4_ST_LD": { + "mean": 332.10231570312504, + "std": 38.62312145359149 + }, + "P4_ST_PO": { + "mean": 332.3768842187501, + "std": 35.29976090158868 + }, + "P4_ST_PT01": { + "mean": 10046.07291666667, + "std": 26.886181059064928 + }, + "P4_ST_TT01": { + "mean": 27608.219752005196, + "std": 24.726020111517904 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.020272347909024324, + "std_abs_err": 0.017998437082092405 + }, + "P1_B2016": { + "mean_abs_err": 0.0002865138768999298, + "std_abs_err": 0.0036854222628889544 + }, + "P1_B3004": { + "mean_abs_err": 7.383953824526145, + "std_abs_err": 15.41814958804181 + }, + "P1_B3005": { + "mean_abs_err": 45.609026528629784, + "std_abs_err": 61.18331651100028 + }, + "P1_B4002": { + "mean_abs_err": 1.2268236801201198, + "std_abs_err": 0.5786734991990778 + }, + "P1_B4005": { + "mean_abs_err": 60.52510701619792, + "std_abs_err": 42.18129744770969 + }, + "P1_B400B": { + "mean_abs_err": 3.88132569880122, + "std_abs_err": 2.3957118082075795 + }, + "P1_B4022": { + "mean_abs_err": 0.8204088942641405, + "std_abs_err": 0.5303998331235046 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.259963494222273, + "std_abs_err": 0.10685096233597591 + }, + "P1_FCV03D": { + "mean_abs_err": 0.003253944986468582, + "std_abs_err": 0.04135018232286569 + }, + "P1_FCV03Z": { + "mean_abs_err": 0.25732847706488826, + "std_abs_err": 0.1748412250667064 + }, + "P1_FT01": { + "mean_abs_err": 0.11856202392803539, + "std_abs_err": 4.694296853120999 + }, + "P1_FT01Z": { + "mean_abs_err": 2.0127761656115126, + "std_abs_err": 8.844088383985039 + }, + "P1_FT02": { + "mean_abs_err": 5.224366984626158, + "std_abs_err": 0.8026135871278939 + }, + "P1_FT02Z": { + "mean_abs_err": 650.8846603586487, + "std_abs_err": 303.34657833260906 + }, + "P1_FT03": { + "mean_abs_err": 0.07835988560412943, + "std_abs_err": 0.28733499161159415 + }, + "P1_FT03Z": { + "mean_abs_err": 778.4814279578968, + "std_abs_err": 27.36394076071023 + }, + "P1_LCV01D": { + "mean_abs_err": 0.013228093433202304, + "std_abs_err": 0.13131438578316734 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.363608091837925, + "std_abs_err": 0.13047464281191168 + }, + "P1_LIT01": { + "mean_abs_err": 0.0751003140480293, + "std_abs_err": 0.237645000281983 + }, + "P1_PCV01D": { + "mean_abs_err": 0.015844490899667107, + "std_abs_err": 0.08340833752946608 + }, + "P1_PCV01Z": { + "mean_abs_err": 0.07288393231921475, + "std_abs_err": 0.08938785242306224 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.07670294660492694, + "std_abs_err": 0.09684446546464737 + }, + "P1_PIT01": { + "mean_abs_err": 0.008466090545058869, + "std_abs_err": 0.023453711530955344 + }, + "P1_PIT02": { + "mean_abs_err": 0.26197487679985004, + "std_abs_err": 0.44825649884554725 + }, + "P1_TIT01": { + "mean_abs_err": 0.01964066430818434, + "std_abs_err": 0.0008864832074719775 + }, + "P1_TIT02": { + "mean_abs_err": 0.024116647086550813, + "std_abs_err": 0.01659305154097601 + }, + "P2_24Vdc": { + "mean_abs_err": 9.712450266263772e-05, + "std_abs_err": 0.0001967416370891339 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.515617804529029, + "std_abs_err": 1.5791291615210525 + }, + "P2_HILout": { + "mean_abs_err": 0.023249037195910205, + "std_abs_err": 0.9656627941268248 + }, + "P2_MSD": { + "mean_abs_err": 4.750739999999951, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.8544382467767946, + "std_abs_err": 0.6824309747896598 + }, + "P2_SIT02": { + "mean_abs_err": 0.020525789328075916, + "std_abs_err": 0.6495131289229921 + }, + "P2_VT01": { + "mean_abs_err": 3.8146010144402e-05, + "std_abs_err": 0.00047310505737040154 + }, + "P2_VXT02": { + "mean_abs_err": 0.0008751650983298553, + "std_abs_err": 0.006324177978086931 + }, + "P2_VXT03": { + "mean_abs_err": 0.0012150314563443843, + "std_abs_err": 0.007694179692397607 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006416364512675798, + "std_abs_err": 0.006064570246843981 + }, + "P2_VYT03": { + "mean_abs_err": 0.0009654513395496167, + "std_abs_err": 0.009335301395971995 + }, + "P3_FIT01": { + "mean_abs_err": 165.64798433534963, + "std_abs_err": 69.0480032867315 + }, + "P3_LCP01D": { + "mean_abs_err": 6.145404988596965, + "std_abs_err": 13.480129956998098 + }, + "P3_LCV01D": { + "mean_abs_err": 39.1848576843704, + "std_abs_err": 33.81202610489072 + }, + "P3_LIT01": { + "mean_abs_err": 2.423402303367766, + "std_abs_err": 17.05234267421838 + }, + "P3_PIT01": { + "mean_abs_err": 529.6760029972861, + "std_abs_err": 454.19934303030504 + }, + "P4_HT_FD": { + "mean_abs_err": 0.0003587912213149268, + "std_abs_err": 0.0023800310237707483 + }, + "P4_HT_LD": { + "mean_abs_err": 149.69291447551893, + "std_abs_err": 1.8494828916530537 + }, + "P4_HT_PO": { + "mean_abs_err": 0.013792804201415265, + "std_abs_err": 0.10343612615498898 + }, + "P4_LD": { + "mean_abs_err": 0.004436867110882758, + "std_abs_err": 0.4162762949210119 + }, + "P4_ST_FD": { + "mean_abs_err": 3.932427011685123e-05, + "std_abs_err": 0.002404945680578129 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.683282736845285, + "std_abs_err": 45.00675889595482 + }, + "P4_ST_LD": { + "mean_abs_err": 0.09941853330735739, + "std_abs_err": 0.76995216197394 + }, + "P4_ST_PO": { + "mean_abs_err": 0.1079128950913173, + "std_abs_err": 0.8488173991803762 + }, + "P4_ST_PT01": { + "mean_abs_err": 4.502240152412924, + "std_abs_err": 9.101411033745798 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.8129396743352117, + "std_abs_err": 0.8086983069495766 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 1.0, + "P1_B2016": 0.04335, + "P1_B3004": 0.8108, + "P1_B3005": 0.8106, + "P1_B4002": 0.81055, + "P1_B4005": 0.9113, + "P1_B400B": 0.4626916666666666, + "P1_B4022": 0.5730333333333334, + "P1_FCV02Z": 0.7251208333333333, + "P1_FCV03D": 0.48788750000000003, + "P1_FCV03Z": 0.4417083333333333, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4750083333333333, + "P1_FT02": 0.45717083333333336, + "P1_FT02Z": 0.8361000000000001, + "P1_FT03": 0.5241, + "P1_FT03Z": 1.0, + "P1_LCV01D": 0.4472333333333333, + "P1_LCV01Z": 0.413675, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5574625, + "P1_PCV01Z": 0.5587541666666667, + "P1_PCV02Z": 1.0, + "P1_PIT01": 0.15811666666666668, + "P1_PIT02": 0.61435, + "P1_TIT01": 0.1049416666666666, + "P1_TIT02": 0.10090416666666668, + "P2_24Vdc": 0.11985833333333334, + "P2_CO_rpm": 0.09815833333333329, + "P2_HILout": 0.08234999999999998, + "P2_MSD": 1.0, + "P2_SIT01": 0.45226666666666665, + "P2_SIT02": 0.5010125, + "P2_VT01": 0.12266250000000001, + "P2_VXT02": 0.28207916666666666, + "P2_VXT03": 0.43662083333333335, + "P2_VYT02": 0.25022083333333334, + "P2_VYT03": 0.26955416666666665, + "P3_FIT01": 0.23422916666666668, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.0899583333333333, + "P3_LIT01": 0.09349166666666664, + "P3_PIT01": 0.4869791666666667, + "P4_HT_FD": 0.17737083333333337, + "P4_HT_LD": 0.9375, + "P4_HT_PO": 0.09085833333333332, + "P4_LD": 0.031137499999999985, + "P4_ST_FD": 0.026145833333333424, + "P4_ST_GOV": 0.03512499999999996, + "P4_ST_LD": 0.03265416666666665, + "P4_ST_PO": 0.03597499999999998, + "P4_ST_PT01": 0.13089166666666663, + "P4_ST_TT01": 0.08401666666666663 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.08333, + "q25_diff": 0.012630000000000002, + "q50_diff": 0.012630000000000002, + "q75_diff": 0.012630000000000002, + "q95_diff": 0.012630000000000002 + }, + "P1_B2016": { + "q05_diff": 0.01150999999999991, + "q25_diff": 0.010229999999999961, + "q50_diff": 0.005260000000000042, + "q75_diff": 0.014129999999999976, + "q95_diff": 0.030100000000000016 + }, + "P1_B3004": { + "q05_diff": 2.8398500000000126, + "q25_diff": 1.6357399999999984, + "q50_diff": 1.6357399999999984, + "q75_diff": 1.6357399999999984, + "q95_diff": 1.6357399999999984 + }, + "P1_B3005": { + "q05_diff": 13.44177000000002, + "q25_diff": 13.44177000000002, + "q50_diff": 13.44177000000002, + "q75_diff": 13.44177000000002, + "q95_diff": 99.67999000000009 + }, + "P1_B4002": { + "q05_diff": 1.5437500000000028, + "q25_diff": 0.11175000000000068, + "q50_diff": 0.11175000000000068, + "q75_diff": 0.11175000000000068, + "q95_diff": 0.11175000000000068 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 100.0, + "q50_diff": 100.0, + "q75_diff": 100.0, + "q95_diff": 100.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2480.07193, + "q50_diff": 189.2133799999997, + "q75_diff": 9.826660000000174, + "q95_diff": 7.159419999999955 + }, + "P1_B4022": { + "q05_diff": 1.2294699999999992, + "q25_diff": 0.1574500000000043, + "q50_diff": 0.4875299999999996, + "q75_diff": 0.5946499999999943, + "q95_diff": 0.6461800000000011 + }, + "P1_FCV02Z": { + "q05_diff": 0.015249999999999986, + "q25_diff": 0.007630000000000026, + "q50_diff": 0.03815000000000013, + "q75_diff": 40.39002, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 2.9137799999999956, + "q25_diff": 0.10161999999999694, + "q50_diff": 8.544129999999996, + "q75_diff": 11.150819999999996, + "q95_diff": 0.6586299999999881 + }, + "P1_FCV03Z": { + "q05_diff": 3.2806399999999982, + "q25_diff": 0.022889999999996746, + "q50_diff": 8.96454, + "q75_diff": 10.97869, + "q95_diff": 0.11443999999998766 + }, + "P1_FT01": { + "q05_diff": 7.82011, + "q25_diff": 8.583240000000018, + "q50_diff": 32.806329999999974, + "q75_diff": 37.95618999999999, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 39.27484000000004, + "q25_diff": 20.004209999999944, + "q50_diff": 74.79840000000002, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.2493299999999, + "q50_diff": 270.11572, + "q75_diff": 12.970209999999952, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 27.82217, + "q25_diff": 2761.19012, + "q50_diff": 1129.6900699999999, + "q75_diff": 870.8322700000001, + "q95_diff": 865.2337600000001 + }, + "P1_FT03": { + "q05_diff": 35.85419999999999, + "q25_diff": 1.525890000000004, + "q50_diff": 51.52633000000003, + "q75_diff": 59.73392999999999, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 788.58806, + "q25_diff": 752.67282, + "q50_diff": 703.89301, + "q75_diff": 697.63843, + "q95_diff": 800.17028 + }, + "P1_LCV01D": { + "q05_diff": 0.10132999999999992, + "q25_diff": 1.2110500000000002, + "q50_diff": 4.63297, + "q75_diff": 3.3106399999999994, + "q95_diff": 0.6154100000000007 + }, + "P1_LCV01Z": { + "q05_diff": 0.1068100000000003, + "q25_diff": 1.1901799999999998, + "q50_diff": 4.753120000000001, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.7095400000000005 + }, + "P1_LIT01": { + "q05_diff": 14.370849999999962, + "q25_diff": 1.0231899999999996, + "q50_diff": 6.26075000000003, + "q75_diff": 8.229829999999993, + "q95_diff": 34.80608999999998 + }, + "P1_PCV01D": { + "q05_diff": 1.0574200000000005, + "q25_diff": 3.7118600000000015, + "q50_diff": 22.24167, + "q75_diff": 44.863440000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 1.0551200000000023, + "q25_diff": 3.814699999999995, + "q50_diff": 22.72428, + "q75_diff": 45.34911, + "q95_diff": 45.14223 + }, + "P1_PCV02Z": { + "q05_diff": 0.2441399999999998, + "q25_diff": 0.2441399999999998, + "q50_diff": 0.2517700000000005, + "q75_diff": 0.2517700000000005, + "q95_diff": 0.2517700000000005 + }, + "P1_PIT01": { + "q05_diff": 0.016780000000000017, + "q25_diff": 0.009160000000000057, + "q50_diff": 0.009919999999999929, + "q75_diff": 0.02364999999999995, + "q95_diff": 0.008850000000000025 + }, + "P1_PIT02": { + "q05_diff": 0.02975, + "q25_diff": 0.10069999999999998, + "q50_diff": 0.10146999999999998, + "q75_diff": 0.0015300000000000313, + "q95_diff": 0.0022900000000000142 + }, + "P1_TIT01": { + "q05_diff": 0.030509999999999593, + "q25_diff": 0.06103999999999843, + "q50_diff": 0.12207000000000079, + "q75_diff": 0.22887999999999664, + "q95_diff": 0.01525999999999783 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.01525999999999783, + "q50_diff": 0.15258999999999645, + "q75_diff": 0.30516999999999683, + "q95_diff": 0.4577500000000043 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008100000000013097, + "q50_diff": 7.000000000090267e-05, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.000920000000000698 + }, + "P2_CO_rpm": { + "q05_diff": 10.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 3.0, + "q95_diff": 1.0 + }, + "P2_HILout": { + "q05_diff": 0.5690999999999349, + "q25_diff": 2.0201399999999694, + "q50_diff": 0.09100000000000819, + "q75_diff": 1.5365600000000086, + "q95_diff": 1.212470000000053 + }, + "P2_MSD": { + "q05_diff": 4.750739999999951, + "q25_diff": 4.750739999999951, + "q50_diff": 4.750739999999951, + "q75_diff": 4.750739999999951, + "q95_diff": 4.750739999999951 + }, + "P2_SIT01": { + "q05_diff": 2.0, + "q25_diff": 3.0, + "q50_diff": 5.0, + "q75_diff": 5.0, + "q95_diff": 6.0 + }, + "P2_SIT02": { + "q05_diff": 2.664800000000014, + "q25_diff": 3.5689700000000357, + "q50_diff": 5.617560000000026, + "q75_diff": 6.355469999999968, + "q95_diff": 6.8497899999999845 + }, + "P2_VT01": { + "q05_diff": 0.008340000000000458, + "q25_diff": 0.015229999999998967, + "q50_diff": 0.013970000000000482, + "q75_diff": 0.013349999999999085, + "q95_diff": 0.01905000000000001 + }, + "P2_VXT02": { + "q05_diff": 0.13142999999999994, + "q25_diff": 0.29942999999999964, + "q50_diff": 0.2856899999999998, + "q75_diff": 0.26678000000000024, + "q95_diff": 0.45733999999999986 + }, + "P2_VXT03": { + "q05_diff": 0.09997999999999996, + "q25_diff": 0.26918, + "q50_diff": 0.29715, + "q75_diff": 0.32714, + "q95_diff": 0.61093 + }, + "P2_VYT02": { + "q05_diff": 0.09633000000000003, + "q25_diff": 0.2622200000000001, + "q50_diff": 0.22737000000000007, + "q75_diff": 0.23548999999999953, + "q95_diff": 0.38881999999999994 + }, + "P2_VYT03": { + "q05_diff": 0.05736000000000008, + "q25_diff": 0.18581000000000003, + "q50_diff": 0.1759599999999999, + "q75_diff": 0.18310000000000048, + "q95_diff": 0.35397999999999996 + }, + "P3_FIT01": { + "q05_diff": 3.0, + "q25_diff": 1.0, + "q50_diff": 68.0, + "q75_diff": 246.0, + "q95_diff": 63.0 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 97.14648000000011, + "q75_diff": 135.81640999999945, + "q95_diff": 10.108399999999165 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 256.0, + "q50_diff": 4000.0, + "q75_diff": 174.91797000000042, + "q95_diff": 24.983399999999165 + }, + "P3_LIT01": { + "q05_diff": 1133.4458000000004, + "q25_diff": 26.34081999999944, + "q50_diff": 646.6298800000004, + "q75_diff": 1146.7011700000003, + "q95_diff": 3.5078099999991537 + }, + "P3_PIT01": { + "q05_diff": 4.0, + "q25_diff": 5.0, + "q50_diff": 1.0, + "q75_diff": 927.0, + "q95_diff": 3523.0 + }, + "P4_HT_FD": { + "q05_diff": 6.999999999999997e-05, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 7e-05, + "q95_diff": 0.0 + }, + "P4_HT_LD": { + "q05_diff": 124.7862, + "q25_diff": 124.7862, + "q50_diff": 179.53157, + "q75_diff": 193.36982, + "q95_diff": 56.57957999999999 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 1.30212, + "q50_diff": 14.764660000000003, + "q75_diff": 0.74512, + "q95_diff": 0.09387999999999863 + }, + "P4_LD": { + "q05_diff": 0.7181400000000053, + "q25_diff": 3.1035400000000095, + "q50_diff": 3.4634100000000103, + "q75_diff": 0.2097699999999918, + "q95_diff": 4.065030000000036 + }, + "P4_ST_FD": { + "q05_diff": 6.999999999999975e-05, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 7.000000000000008e-05, + "q95_diff": 0.00014999999999999996 + }, + "P4_ST_GOV": { + "q05_diff": 127.6191400000007, + "q25_diff": 38.41602000000057, + "q50_diff": 32.93359000000055, + "q75_diff": 105.15038999999888, + "q95_diff": 410.19530999999915 + }, + "P4_ST_LD": { + "q05_diff": 0.6897000000000162, + "q25_diff": 0.39995999999996457, + "q50_diff": 0.44855000000001155, + "q75_diff": 2.0376299999999787, + "q95_diff": 7.642819999999972 + }, + "P4_ST_PO": { + "q05_diff": 2.139160000000004, + "q25_diff": 0.7237599999999702, + "q50_diff": 0.5847499999999854, + "q75_diff": 2.223230000000001, + "q95_diff": 7.533959999999979 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 20.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 7.0, + "q50_diff": 1.8574200000002747, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0189476811306442e-05, + "P1_B2016": 0.10071267986625787, + "P1_B3004": 1.018740504876714e-05, + "P1_B3005": 0.9999898125942444, + "P1_B4002": 0.9999898146591784, + "P1_B4005": 0.9999997508214005, + "P1_B400B": 0.3960311765090463, + "P1_B4022": 0.03037469907643997, + "P1_FCV02Z": 0.06868229817924654, + "P1_FCV03D": 0.2646213594413662, + "P1_FCV03Z": 0.14205582255747784, + "P1_FT01": 0.6239591138354834, + "P1_FT01Z": 0.33564561874055543, + "P1_FT02": 0.6708616495569188, + "P1_FT02Z": 0.6709166281668304, + "P1_FT03": 0.3714260732750322, + "P1_FT03Z": 0.3712890162304253, + "P1_LCV01D": 0.23960508118025903, + "P1_LCV01Z": 0.19275418578808967, + "P1_LIT01": 0.11634130732591486, + "P1_PCV01D": 0.13830931572626504, + "P1_PCV01Z": 0.10517715317199072, + "P1_PCV02Z": 0.5669400491575541, + "P1_PIT01": 0.45772096455158384, + "P1_PIT02": 0.2070476045979056, + "P1_TIT01": 0.045703475541319594, + "P1_TIT02": 0.06964659147235452, + "P2_24Vdc": 0.13961252605352986, + "P2_CO_rpm": 0.39373561196907797, + "P2_HILout": 0.2651520495156722, + "P2_MSD": 0.0, + "P2_SIT01": 0.39211080478497423, + "P2_SIT02": 0.5217236125717422, + "P2_VT01": 0.14516088341296318, + "P2_VXT02": 0.16059773528908394, + "P2_VXT03": 0.18775925906672053, + "P2_VYT02": 0.13988751859181536, + "P2_VYT03": 0.16006325768899177, + "P3_FIT01": 0.10625104442177546, + "P3_LCP01D": 0.2117437883872706, + "P3_LCV01D": 0.28725377371190874, + "P3_LIT01": 0.07064394828613063, + "P3_PIT01": 0.9015472595033223, + "P4_HT_FD": 0.1802507089580071, + "P4_HT_LD": 0.15670838558697175, + "P4_HT_PO": 0.09810051050171509, + "P4_LD": 0.06334103304247474, + "P4_ST_FD": 0.3167365978515703, + "P4_ST_GOV": 0.10930956607198761, + "P4_ST_LD": 0.14894593659591937, + "P4_ST_PO": 0.08522373847555897, + "P4_ST_PT01": 0.6258553186536182, + "P4_ST_TT01": 0.0681936940158665 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.5843189700185825, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + }, + "avg_ks": 0.4039510220125786, + "avg_jsd": 0.02930392024585336, + "avg_lag1_diff": 0.28531566380969187 +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_residual_stat__seed1337/eval_post.json b/example/results/ablations/runs/no_residual_stat__seed1337/eval_post.json new file mode 100644 index 0000000..e58945a --- /dev/null +++ b/example/results/ablations/runs/no_residual_stat__seed1337/eval_post.json @@ -0,0 +1,971 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09355486979166669, + "std": 0.01799194293084728 + }, + "P1_B2016": { + "mean": 1.391552838541667, + "std": 0.13945898473945204 + }, + "P1_B3004": { + "mean": 403.2905014583335, + "std": 14.217563527951576 + }, + "P1_B3005": { + "mean": 1059.5253898958326, + "std": 58.42587703751758 + }, + "P1_B4002": { + "mean": 32.37490617187499, + "std": 0.6519533213854061 + }, + "P1_B4005": { + "mean": 64.99571291666655, + "std": 42.03467331899641 + }, + "P1_B400B": { + "mean": 1774.5945444791669, + "std": 1195.7479847272803 + }, + "P1_B4022": { + "mean": 36.08671476562498, + "std": 0.7147782383668128 + }, + "P1_FCV02Z": { + "mean": 21.954832630208315, + "std": 38.87335152465215 + }, + "P1_FCV03D": { + "mean": 58.14541601562505, + "std": 6.267267768996455 + }, + "P1_FCV03Z": { + "mean": 58.21398343750003, + "std": 6.423211358701469 + }, + "P1_FT01": { + "mean": 194.8422163802083, + "std": 32.042278691765326 + }, + "P1_FT01Z": { + "mean": 877.036905338541, + "std": 84.60979134730329 + }, + "P1_FT02": { + "mean": 1127.59378625, + "std": 889.9987774385405 + }, + "P1_FT02Z": { + "mean": 1843.7261292187509, + "std": 1190.4241853713818 + }, + "P1_FT03": { + "mean": 282.5606103906248, + "std": 33.7615481029071 + }, + "P1_FT03Z": { + "mean": 1058.2564329947922, + "std": 59.50957255113764 + }, + "P1_LCV01D": { + "mean": 11.699055963541667, + "std": 3.0155193220433567 + }, + "P1_LCV01Z": { + "mean": 11.627510442708333, + "std": 3.0251072332459787 + }, + "P1_LIT01": { + "mean": 404.03977184895825, + "std": 15.869543048655954 + }, + "P1_PCV01D": { + "mean": 65.64593611979173, + "std": 23.56629129628467 + }, + "P1_PCV01Z": { + "mean": 65.565628671875, + "std": 23.790956943468952 + }, + "P1_PCV02Z": { + "mean": 11.867279765625007, + "std": 0.10811550492611105 + }, + "P1_PIT01": { + "mean": 1.369015130208333, + "std": 0.08488627090083745 + }, + "P1_PIT02": { + "mean": 0.22518221354166668, + "std": 0.05045513920933258 + }, + "P1_TIT01": { + "mean": 35.66156757812503, + "std": 0.584167110393421 + }, + "P1_TIT02": { + "mean": 36.50086203124998, + "std": 1.238351777565239 + }, + "P2_24Vdc": { + "mean": 28.02860809895833, + "std": 0.0032970984602375326 + }, + "P2_CO_rpm": { + "mean": 54102.19458007815, + "std": 24.034860936309418 + }, + "P2_HILout": { + "mean": 711.4986896354167, + "std": 7.856828438379631 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.0494791666662, + "std": 5.881346335233885 + }, + "P2_SIT02": { + "mean": 783.9242636718743, + "std": 5.840092885735959 + }, + "P2_VT01": { + "mean": 11.92377749999999, + "std": 0.06898678712937246 + }, + "P2_VXT02": { + "mean": -3.2993033072916664, + "std": 0.4773571042493072 + }, + "P2_VXT03": { + "mean": -1.3056782552083335, + "std": 0.35732155885605127 + }, + "P2_VYT02": { + "mean": 3.9808272135416694, + "std": 0.5059750232396318 + }, + "P2_VYT03": { + "mean": 6.233439661458335, + "std": 0.3891382633670636 + }, + "P3_FIT01": { + "mean": 985.2521917708335, + "std": 1710.6766945035433 + }, + "P3_LCP01D": { + "mean": 4590.872553229171, + "std": 5143.480127672877 + }, + "P3_LCV01D": { + "mean": 7094.691859296871, + "std": 6873.739921920712 + }, + "P3_LIT01": { + "mean": 13352.421090390619, + "std": 4371.226979602155 + }, + "P3_PIT01": { + "mean": 119.68190002604165, + "std": 696.7225876768903 + }, + "P4_HT_FD": { + "mean": -0.0004619791666666666, + "std": 0.004412049338174849 + }, + "P4_HT_LD": { + "mean": 34.231568880208286, + "std": 33.60361854009684 + }, + "P4_HT_PO": { + "mean": 35.25451283854166, + "std": 31.650898099038645 + }, + "P4_LD": { + "mean": 366.12431692708344, + "std": 61.03208266436402 + }, + "P4_ST_FD": { + "mean": -6.843750000000004e-05, + "std": 0.004047121938568158 + }, + "P4_ST_GOV": { + "mean": 17918.64454398437, + "std": 1897.1499748837282 + }, + "P4_ST_LD": { + "mean": 332.10231570312504, + "std": 38.62312145359149 + }, + "P4_ST_PO": { + "mean": 332.3768842187501, + "std": 35.29976090158868 + }, + "P4_ST_PT01": { + "mean": 10046.07291666667, + "std": 26.886181059064928 + }, + "P4_ST_TT01": { + "mean": 27608.219752005196, + "std": 24.726020111517904 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 1.2782299308983225e-05, + "std_abs_err": 6.494151245123314e-06 + }, + "P1_B2016": { + "mean_abs_err": 0.0002865138768999298, + "std_abs_err": 0.0036854222628889544 + }, + "P1_B3004": { + "mean_abs_err": 0.09556236619266656, + "std_abs_err": 1.2005860600902345 + }, + "P1_B3005": { + "mean_abs_err": 1.5233966327971302, + "std_abs_err": 2.757439473482698 + }, + "P1_B4002": { + "mean_abs_err": 0.05797985199510691, + "std_abs_err": 0.07327982218632823 + }, + "P1_B4005": { + "mean_abs_err": 4.470605900468627, + "std_abs_err": 0.14662412871327746 + }, + "P1_B400B": { + "mean_abs_err": 3.88132569880122, + "std_abs_err": 2.3957118082075795 + }, + "P1_B4022": { + "mean_abs_err": 0.02554011822245883, + "std_abs_err": 0.04332738353199439 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.996102270263961, + "std_abs_err": 1.5651339146060295 + }, + "P1_FCV03D": { + "mean_abs_err": 0.42244743522177686, + "std_abs_err": 0.3322279497122995 + }, + "P1_FCV03Z": { + "mean_abs_err": 1.0850557426898035, + "std_abs_err": 0.29159260563013323 + }, + "P1_FT01": { + "mean_abs_err": 0.11856202392803539, + "std_abs_err": 4.694296853120999 + }, + "P1_FT01Z": { + "mean_abs_err": 2.0127761656115126, + "std_abs_err": 8.844088383985039 + }, + "P1_FT02": { + "mean_abs_err": 5.224366984626158, + "std_abs_err": 0.8026135871278939 + }, + "P1_FT02Z": { + "mean_abs_err": 65.24768261010217, + "std_abs_err": 2.9211703997677887 + }, + "P1_FT03": { + "mean_abs_err": 0.07835988560412943, + "std_abs_err": 0.28733499161159415 + }, + "P1_FT03Z": { + "mean_abs_err": 2.785605353729352, + "std_abs_err": 1.6159163124796905 + }, + "P1_LCV01D": { + "mean_abs_err": 0.45527669823346884, + "std_abs_err": 0.1009043690132354 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.33856358662958463, + "std_abs_err": 0.061637735900667234 + }, + "P1_LIT01": { + "mean_abs_err": 0.0751003140480293, + "std_abs_err": 0.237645000281983 + }, + "P1_PCV01D": { + "mean_abs_err": 1.8196376158996515, + "std_abs_err": 0.20347540462090308 + }, + "P1_PCV01Z": { + "mean_abs_err": 2.842523125027583, + "std_abs_err": 0.07364286719469249 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.024526819020080026, + "std_abs_err": 0.011271039461463683 + }, + "P1_PIT01": { + "mean_abs_err": 0.008466090545058869, + "std_abs_err": 0.023453711530955344 + }, + "P1_PIT02": { + "mean_abs_err": 0.26197487679985004, + "std_abs_err": 0.44825649884554725 + }, + "P1_TIT01": { + "mean_abs_err": 0.01964066430818434, + "std_abs_err": 0.0008864832074719775 + }, + "P1_TIT02": { + "mean_abs_err": 0.024116647086550813, + "std_abs_err": 0.01659305154097601 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00010160366933931186, + "std_abs_err": 4.536765416938244e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.515617804529029, + "std_abs_err": 1.5791291615210525 + }, + "P2_HILout": { + "mean_abs_err": 0.035880365320508645, + "std_abs_err": 0.5978974388765312 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.8544382467767946, + "std_abs_err": 0.6824309747896598 + }, + "P2_SIT02": { + "mean_abs_err": 0.020525789328075916, + "std_abs_err": 0.6495131289229921 + }, + "P2_VT01": { + "mean_abs_err": 3.8146010144402e-05, + "std_abs_err": 0.00047310505737040154 + }, + "P2_VXT02": { + "mean_abs_err": 0.0008751650983298553, + "std_abs_err": 0.006324177978086931 + }, + "P2_VXT03": { + "mean_abs_err": 0.0012150314563443843, + "std_abs_err": 0.007694179692397607 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006416364512675798, + "std_abs_err": 0.006064570246843981 + }, + "P2_VYT03": { + "mean_abs_err": 0.0009654513395496167, + "std_abs_err": 0.009335301395971995 + }, + "P3_FIT01": { + "mean_abs_err": 165.64798433534963, + "std_abs_err": 69.0480032867315 + }, + "P3_LCP01D": { + "mean_abs_err": 6.145404988596965, + "std_abs_err": 13.480129956998098 + }, + "P3_LCV01D": { + "mean_abs_err": 39.1848576843704, + "std_abs_err": 33.81202610489072 + }, + "P3_LIT01": { + "mean_abs_err": 2.423402303367766, + "std_abs_err": 17.05234267421838 + }, + "P3_PIT01": { + "mean_abs_err": 529.6760029972861, + "std_abs_err": 454.19934303030504 + }, + "P4_HT_FD": { + "mean_abs_err": 0.0003587912213149268, + "std_abs_err": 0.0023800310237707483 + }, + "P4_HT_LD": { + "mean_abs_err": 0.23665180052267232, + "std_abs_err": 0.17982503009853446 + }, + "P4_HT_PO": { + "mean_abs_err": 1.2707050437847514, + "std_abs_err": 0.21002688171716954 + }, + "P4_LD": { + "mean_abs_err": 0.004436867110882758, + "std_abs_err": 0.4162762949210119 + }, + "P4_ST_FD": { + "mean_abs_err": 3.932427011685123e-05, + "std_abs_err": 0.002404945680578129 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.683282736845285, + "std_abs_err": 45.00675889595482 + }, + "P4_ST_LD": { + "mean_abs_err": 0.09941853330735739, + "std_abs_err": 0.76995216197394 + }, + "P4_ST_PO": { + "mean_abs_err": 0.1079128950913173, + "std_abs_err": 0.8488173991803762 + }, + "P4_ST_PT01": { + "mean_abs_err": 4.502240152412924, + "std_abs_err": 9.101411033745798 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.8129396743352117, + "std_abs_err": 0.8086983069495766 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.6595583333333334, + "P1_B2016": 0.04335, + "P1_B3004": 0.671875, + "P1_B3005": 0.5293, + "P1_B4002": 0.7923208333333334, + "P1_B4005": 0.3062625, + "P1_B400B": 0.4626916666666666, + "P1_B4022": 0.5756333333333334, + "P1_FCV02Z": 0.7746, + "P1_FCV03D": 0.45162500000000005, + "P1_FCV03Z": 0.40985833333333327, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4750083333333333, + "P1_FT02": 0.45717083333333336, + "P1_FT02Z": 0.4029958333333334, + "P1_FT03": 0.5241, + "P1_FT03Z": 0.46950833333333336, + "P1_LCV01D": 0.3859916666666667, + "P1_LCV01Z": 0.397625, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5304583333333333, + "P1_PCV01Z": 0.5184416666666667, + "P1_PCV02Z": 0.6588541666666666, + "P1_PIT01": 0.15811666666666668, + "P1_PIT02": 0.61435, + "P1_TIT01": 0.1049416666666666, + "P1_TIT02": 0.10090416666666668, + "P2_24Vdc": 0.13548333333333334, + "P2_CO_rpm": 0.09815833333333329, + "P2_HILout": 0.08242499999999997, + "P2_MSD": 0.0, + "P2_SIT01": 0.45226666666666665, + "P2_SIT02": 0.5010125, + "P2_VT01": 0.12266250000000001, + "P2_VXT02": 0.28207916666666666, + "P2_VXT03": 0.43662083333333335, + "P2_VYT02": 0.25022083333333334, + "P2_VYT03": 0.26955416666666665, + "P3_FIT01": 0.23422916666666668, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.0899583333333333, + "P3_LIT01": 0.09349166666666664, + "P3_PIT01": 0.4869791666666667, + "P4_HT_FD": 0.17737083333333337, + "P4_HT_LD": 0.06742500000000001, + "P4_HT_PO": 0.05970416666666667, + "P4_LD": 0.031137499999999985, + "P4_ST_FD": 0.026145833333333424, + "P4_ST_GOV": 0.03512499999999996, + "P4_ST_LD": 0.03265416666666665, + "P4_ST_PO": 0.03597499999999998, + "P4_ST_PT01": 0.13089166666666663, + "P4_ST_TT01": 0.08401666666666663 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.02845, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0011099999999999999, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.01150999999999991, + "q25_diff": 0.010229999999999961, + "q50_diff": 0.005260000000000042, + "q75_diff": 0.014129999999999976, + "q95_diff": 0.030100000000000016 + }, + "P1_B3004": { + "q05_diff": 14.80313000000001, + "q25_diff": 1.6197799999999916, + "q50_diff": 6.726189999999974, + "q75_diff": 8.508819999999957, + "q95_diff": 37.86444 + }, + "P1_B3005": { + "q05_diff": 40.40789999999993, + "q25_diff": 8.045049999999947, + "q50_diff": 89.51202, + "q75_diff": 104.97393, + "q95_diff": 4.4593499999998585 + }, + "P1_B4002": { + "q05_diff": 0.4564699999999995, + "q25_diff": 1.6555000000000035, + "q50_diff": 1.6555000000000035, + "q75_diff": 0.9287000000000063, + "q95_diff": 0.16969000000000278 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 81.96193, + "q50_diff": 0.04000000000000625, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2480.07193, + "q50_diff": 189.2133799999997, + "q75_diff": 9.826660000000174, + "q95_diff": 7.159419999999955 + }, + "P1_B4022": { + "q05_diff": 0.38222999999999985, + "q25_diff": 1.3479000000000028, + "q50_diff": 1.4273799999999994, + "q75_diff": 0.9127399999999994, + "q95_diff": 0.5270299999999963 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.05340999999999996, + "q75_diff": 38.169869999999996, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 1.3654200000000003, + "q25_diff": 0.2554099999999977, + "q50_diff": 8.60333, + "q75_diff": 10.84863, + "q95_diff": 0.824300000000008 + }, + "P1_FCV03Z": { + "q05_diff": 1.7547599999999974, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.148199999999996, + "q75_diff": 10.864249999999998, + "q95_diff": 0.9307899999999876 + }, + "P1_FT01": { + "q05_diff": 7.82011, + "q25_diff": 8.583240000000018, + "q50_diff": 32.806329999999974, + "q75_diff": 37.95618999999999, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 39.27484000000004, + "q25_diff": 20.004209999999944, + "q50_diff": 74.79840000000002, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.2493299999999, + "q50_diff": 270.11572, + "q75_diff": 12.970209999999952, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 1.9862300000000026, + "q25_diff": 2300.54303, + "q50_diff": 14.318849999999657, + "q75_diff": 7.299800000000232, + "q95_diff": 6.317140000000109 + }, + "P1_FT03": { + "q05_diff": 35.85419999999999, + "q25_diff": 1.525890000000004, + "q50_diff": 51.52633000000003, + "q75_diff": 59.73392999999999, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 23.41980000000001, + "q25_diff": 2.9805299999999306, + "q50_diff": 84.98266000000012, + "q75_diff": 100.6383699999999, + "q95_diff": 3.5079399999999623 + }, + "P1_LCV01D": { + "q05_diff": 0.17497000000000007, + "q25_diff": 0.29953000000000074, + "q50_diff": 4.171619999999999, + "q75_diff": 3.162939999999999, + "q95_diff": 0.7125800000000009 + }, + "P1_LCV01Z": { + "q05_diff": 0.0, + "q25_diff": 0.5874600000000001, + "q50_diff": 4.638670000000001, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.9536700000000007 + }, + "P1_LIT01": { + "q05_diff": 14.370849999999962, + "q25_diff": 1.0231899999999996, + "q50_diff": 6.26075000000003, + "q75_diff": 8.229829999999993, + "q95_diff": 34.80608999999998 + }, + "P1_PCV01D": { + "q05_diff": 0.18625000000000114, + "q25_diff": 2.7490099999999984, + "q50_diff": 18.274499999999996, + "q75_diff": 43.266600000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.22888000000000375, + "q25_diff": 2.0370500000000007, + "q50_diff": 16.029360000000004, + "q75_diff": 43.33495, + "q95_diff": 45.1493 + }, + "P1_PCV02Z": { + "q05_diff": 0.2288800000000002, + "q25_diff": 0.2212499999999995, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_PIT01": { + "q05_diff": 0.016780000000000017, + "q25_diff": 0.009160000000000057, + "q50_diff": 0.009919999999999929, + "q75_diff": 0.02364999999999995, + "q95_diff": 0.008850000000000025 + }, + "P1_PIT02": { + "q05_diff": 0.02975, + "q25_diff": 0.10069999999999998, + "q50_diff": 0.10146999999999998, + "q75_diff": 0.0015300000000000313, + "q95_diff": 0.0022900000000000142 + }, + "P1_TIT01": { + "q05_diff": 0.030509999999999593, + "q25_diff": 0.06103999999999843, + "q50_diff": 0.12207000000000079, + "q75_diff": 0.22887999999999664, + "q95_diff": 0.01525999999999783 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.01525999999999783, + "q50_diff": 0.15258999999999645, + "q75_diff": 0.30516999999999683, + "q95_diff": 0.4577500000000043 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.00013999999999825263, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.00010999999999938836 + }, + "P2_CO_rpm": { + "q05_diff": 10.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 3.0, + "q95_diff": 1.0 + }, + "P2_HILout": { + "q05_diff": 0.3295900000000529, + "q25_diff": 1.6540499999999838, + "q50_diff": 0.31737999999995736, + "q75_diff": 1.7334000000000742, + "q95_diff": 2.2399900000000343 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 2.0, + "q25_diff": 3.0, + "q50_diff": 5.0, + "q75_diff": 5.0, + "q95_diff": 6.0 + }, + "P2_SIT02": { + "q05_diff": 2.664800000000014, + "q25_diff": 3.5689700000000357, + "q50_diff": 5.617560000000026, + "q75_diff": 6.355469999999968, + "q95_diff": 6.8497899999999845 + }, + "P2_VT01": { + "q05_diff": 0.008340000000000458, + "q25_diff": 0.015229999999998967, + "q50_diff": 0.013970000000000482, + "q75_diff": 0.013349999999999085, + "q95_diff": 0.01905000000000001 + }, + "P2_VXT02": { + "q05_diff": 0.13142999999999994, + "q25_diff": 0.29942999999999964, + "q50_diff": 0.2856899999999998, + "q75_diff": 0.26678000000000024, + "q95_diff": 0.45733999999999986 + }, + "P2_VXT03": { + "q05_diff": 0.09997999999999996, + "q25_diff": 0.26918, + "q50_diff": 0.29715, + "q75_diff": 0.32714, + "q95_diff": 0.61093 + }, + "P2_VYT02": { + "q05_diff": 0.09633000000000003, + "q25_diff": 0.2622200000000001, + "q50_diff": 0.22737000000000007, + "q75_diff": 0.23548999999999953, + "q95_diff": 0.38881999999999994 + }, + "P2_VYT03": { + "q05_diff": 0.05736000000000008, + "q25_diff": 0.18581000000000003, + "q50_diff": 0.1759599999999999, + "q75_diff": 0.18310000000000048, + "q95_diff": 0.35397999999999996 + }, + "P3_FIT01": { + "q05_diff": 3.0, + "q25_diff": 1.0, + "q50_diff": 68.0, + "q75_diff": 246.0, + "q95_diff": 63.0 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 97.14648000000011, + "q75_diff": 135.81640999999945, + "q95_diff": 10.108399999999165 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 256.0, + "q50_diff": 4000.0, + "q75_diff": 174.91797000000042, + "q95_diff": 24.983399999999165 + }, + "P3_LIT01": { + "q05_diff": 1133.4458000000004, + "q25_diff": 26.34081999999944, + "q50_diff": 646.6298800000004, + "q75_diff": 1146.7011700000003, + "q95_diff": 3.5078099999991537 + }, + "P3_PIT01": { + "q05_diff": 4.0, + "q25_diff": 5.0, + "q50_diff": 1.0, + "q75_diff": 927.0, + "q95_diff": 3523.0 + }, + "P4_HT_FD": { + "q05_diff": 6.999999999999997e-05, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 7e-05, + "q95_diff": 0.0 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 27.14122, + "q75_diff": 0.13020999999999106, + "q95_diff": 0.21701000000000192 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 0.9223299999999999, + "q50_diff": 8.89752, + "q75_diff": 0.054199999999994475, + "q95_diff": 0.6510700000000043 + }, + "P4_LD": { + "q05_diff": 0.7181400000000053, + "q25_diff": 3.1035400000000095, + "q50_diff": 3.4634100000000103, + "q75_diff": 0.2097699999999918, + "q95_diff": 4.065030000000036 + }, + "P4_ST_FD": { + "q05_diff": 6.999999999999975e-05, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 7.000000000000008e-05, + "q95_diff": 0.00014999999999999996 + }, + "P4_ST_GOV": { + "q05_diff": 127.6191400000007, + "q25_diff": 38.41602000000057, + "q50_diff": 32.93359000000055, + "q75_diff": 105.15038999999888, + "q95_diff": 410.19530999999915 + }, + "P4_ST_LD": { + "q05_diff": 0.6897000000000162, + "q25_diff": 0.39995999999996457, + "q50_diff": 0.44855000000001155, + "q75_diff": 2.0376299999999787, + "q95_diff": 7.642819999999972 + }, + "P4_ST_PO": { + "q05_diff": 2.139160000000004, + "q25_diff": 0.7237599999999702, + "q50_diff": 0.5847499999999854, + "q75_diff": 2.223230000000001, + "q95_diff": 7.533959999999979 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 20.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 7.0, + "q50_diff": 1.8574200000002747, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0624831607922858, + "P1_B2016": 0.10071267986625787, + "P1_B3004": 0.9420514424699565, + "P1_B3005": 0.9755864990985265, + "P1_B4002": 0.9939428775878422, + "P1_B4005": 1.0901635813805555, + "P1_B400B": 0.3960311765090463, + "P1_B4022": 1.0728287547128275, + "P1_FCV02Z": 1.0856923346443097, + "P1_FCV03D": 1.0287616800381814, + "P1_FCV03Z": 1.01920928123416, + "P1_FT01": 0.6239591138354834, + "P1_FT01Z": 0.33564561874055543, + "P1_FT02": 0.6708616495569188, + "P1_FT02Z": 1.0572315062387077, + "P1_FT03": 0.3714260732750322, + "P1_FT03Z": 0.9861265787637634, + "P1_LCV01D": 0.9922897795117445, + "P1_LCV01Z": 0.9035160861736835, + "P1_LIT01": 0.11634130732591486, + "P1_PCV01D": 0.9847169107083952, + "P1_PCV01Z": 1.1483081333334428, + "P1_PCV02Z": 0.5764046458797007, + "P1_PIT01": 0.45772096455158384, + "P1_PIT02": 0.2070476045979056, + "P1_TIT01": 0.045703475541319594, + "P1_TIT02": 0.06964659147235452, + "P2_24Vdc": 0.039965792976989475, + "P2_CO_rpm": 0.39373561196907797, + "P2_HILout": 0.24706059027478758, + "P2_MSD": 0.0, + "P2_SIT01": 0.39211080478497423, + "P2_SIT02": 0.5217236125717422, + "P2_VT01": 0.14516088341296318, + "P2_VXT02": 0.16059773528908394, + "P2_VXT03": 0.18775925906672053, + "P2_VYT02": 0.13988751859181536, + "P2_VYT03": 0.16006325768899177, + "P3_FIT01": 0.10625104442177546, + "P3_LCP01D": 0.2117437883872706, + "P3_LCV01D": 0.28725377371190874, + "P3_LIT01": 0.07064394828613063, + "P3_PIT01": 0.9015472595033223, + "P4_HT_FD": 0.1802507089580071, + "P4_HT_LD": 1.052071148480924, + "P4_HT_PO": 1.0918064404320895, + "P4_LD": 0.06334103304247474, + "P4_ST_FD": 0.3167365978515703, + "P4_ST_GOV": 0.10930956607198761, + "P4_ST_LD": 0.14894593659591937, + "P4_ST_PO": 0.08522373847555897, + "P4_ST_PT01": 0.6258553186536182, + "P4_ST_TT01": 0.0681936940158665 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.5843189700185825, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_snr_weight__seed1337/comprehensive_eval.json b/example/results/ablations/runs/no_snr_weight__seed1337/comprehensive_eval.json new file mode 100644 index 0000000..bda7bbf --- /dev/null +++ b/example/results/ablations/runs/no_snr_weight__seed1337/comprehensive_eval.json @@ -0,0 +1,537 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_snr_weight__seed1337/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.39950668238993714, + "avg_jsd": 0.02154500462890076, + "avg_lag1_diff": 0.2994253248181626 + }, + "two_sample": { + "continuous_mmd_rbf": 0.6485295295715332, + "continuous_mmd_gamma": 1.6569123503075158e-05, + "histogram_mmd_rbf": 0.6485327482223511, + "histogram_mmd_gamma": 1.656833961161223e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 164.53665161132812, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 2.98201284485682, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.21394668519496918, + "corr_frobenius": 16.57740592956543, + "lag1_corr_mean_abs_diff": 0.21236610412597656, + "lag1_corr_frobenius": 16.575729370117188, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.2973121404647827, + "corr_frobenius": 10.449137687683105, + "lag1_corr_mean_abs_diff": 0.3082079291343689, + "lag1_corr_frobenius": 10.682194709777832 + }, + "P2": { + "corr_mean_abs_diff": 0.12466901540756226, + "corr_frobenius": 2.944415807723999, + "lag1_corr_mean_abs_diff": 0.15586906671524048, + "lag1_corr_frobenius": 2.846930980682373 + }, + "P3": { + "corr_mean_abs_diff": 0.4641381800174713, + "corr_frobenius": 3.0637080669403076, + "lag1_corr_mean_abs_diff": 0.5239542126655579, + "lag1_corr_frobenius": 3.1854543685913086 + }, + "P4": { + "corr_mean_abs_diff": 0.18219970166683197, + "corr_frobenius": 2.7139594554901123, + "lag1_corr_mean_abs_diff": 0.20237688720226288, + "lag1_corr_frobenius": 2.932380199432373 + } + } + }, + "frequency": { + "avg_psd_l1": 0.019868340343236923, + "avg_psd_cosine": 0.3614977909231035, + "avg_low_high_ratio_abs_diff": 9433974.92816077 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 15.0, + "mean_dwell": 24.0, + "median_dwell": 2.5, + "mean_step": 68.42660395304362, + "median_step": 1.9043693542480469 + }, + "P1_B2004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B3004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.01115664489586733, + "median_step": 0.004810333251953125 + }, + "P1_B3005": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22998.14285714286, + "mean_abs_diff_mean_dwell": 14201.625685203559, + "mean_abs_diff_median_dwell": 14207.642857142857, + "mean_abs_diff_mean_step": 33.693117622162895, + "mean_abs_diff_median_step": 0.831573486328125 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 1.0, + "change_rate": 0.0, + "step_median": NaN + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.411041259765625, + "mean_abs_diff_change_rate": 0.4608201173921447 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.0026041666666666665, + "top1_mass": 1.0, + "top3_mass": 1.0, + "median_dwell": 384.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8880208333333334, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.078125, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9192708333333334, + "top1_mass": 0.08333333333333333, + "top3_mass": 0.08854166666666667, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.046875, + "top1_mass": 0.4088541666666667, + "top3_mass": 0.6901041666666666, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 1.0, + "top1_mass": 0.0026041666666666665, + "top3_mass": 0.0078125, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.3671875, + "top1_mass": 0.16145833333333334, + "top3_mass": 0.2760416666666667, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.9661458333333334, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.015625, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.19791666666666666, + "top1_mass": 0.22916666666666666, + "top3_mass": 0.4036458333333333, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.519494374593099, + "mean_abs_diff_top1_mass": 0.12373224894205728, + "mean_abs_diff_top3_mass": 0.0999310811360677, + "mean_abs_diff_median_dwell": 47.75 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17409999668598175, + "q50": 0.17409999668598175, + "q95": 0.27557000517845154, + "tail_ratio": NaN + }, + "P2_SIT02": { + "q05": 774.923583984375, + "q50": 784.5628051757812, + "q95": 791.695068359375, + "tail_ratio": 0.7399211037871448 + }, + "P1_FT03": { + "q05": 209.12059020996094, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.19295026347175762 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.644094243645668, + "mean_abs_diff_q50": 2.27042847375075, + "mean_abs_diff_q95": 3.5563383996486664, + "mean_abs_diff_tail_ratio": 0.04706495018632424 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 25.53740243936643, + "median_step": 12.781341552734375 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 782.3384777250744, + "median_step": 406.63330078125 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89282.0, + "mean_abs_diff_mean_dwell": 0.0943341942636573, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 396.286839506705, + "mean_abs_diff_median_step": 206.44041442871094 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 34.0006217956543, + "std": 31.54228401184082, + "lag1": 0.5838339328765869 + }, + "P2_24Vdc": { + "mean": 28.028640747070312, + "std": 0.003505086060613394, + "lag1": 0.1788267344236374 + }, + "P2_HILout": { + "mean": 711.4868774414062, + "std": 9.422904014587402, + "lag1": 0.029019661247730255 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.28165117899576825, + "mean_abs_diff_std": 0.4687187431845814, + "mean_abs_diff_lag1": 0.25562512253721553 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9607057571411133, + "mae": 0.7719733715057373 + }, + "real_plus_synthetic": { + "rmse": 0.5602709650993347, + "mae": 0.3653229773044586 + }, + "rmse_ratio_synth_to_real": 1.720612828892518 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8270871985157698, + "auprc": 0.5839558451818384, + "threshold": 0.009951102547347546, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6666666666666667, + "best_f1_threshold": 1.4928399324417114 + }, + "real_plus_synthetic": { + "auroc": 0.8756029684601114, + "auprc": 0.637313246257706, + "threshold": 0.3299241065979004, + "f1": 0.2723404255319149, + "precision": 0.16753926701570682, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.5474025011062622 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_snr_weight__seed1337/comprehensive_eval_post.json b/example/results/ablations/runs/no_snr_weight__seed1337/comprehensive_eval_post.json new file mode 100644 index 0000000..bfab254 --- /dev/null +++ b/example/results/ablations/runs/no_snr_weight__seed1337/comprehensive_eval_post.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_snr_weight__seed1337/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.23022794723510742, + "continuous_mmd_gamma": 4.8223101163589525e-05, + "histogram_mmd_rbf": 0.230302095413208, + "histogram_mmd_gamma": 4.8215780686001324e-05, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 93.34505462646484, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 1.6917577279834184, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.19277362525463104, + "corr_frobenius": 15.927645683288574, + "lag1_corr_mean_abs_diff": 0.1993265599012375, + "lag1_corr_frobenius": 16.577377319335938, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.3383921682834625, + "corr_frobenius": 11.783731460571289, + "lag1_corr_mean_abs_diff": 0.36134302616119385, + "lag1_corr_frobenius": 12.532601356506348 + }, + "P2": { + "corr_mean_abs_diff": 0.09637951850891113, + "corr_frobenius": 2.3628015518188477, + "lag1_corr_mean_abs_diff": 0.15011084079742432, + "lag1_corr_frobenius": 2.8796932697296143 + }, + "P3": { + "corr_mean_abs_diff": 0.4641381800174713, + "corr_frobenius": 3.0637080669403076, + "lag1_corr_mean_abs_diff": 0.5239542126655579, + "lag1_corr_frobenius": 3.1854543685913086 + }, + "P4": { + "corr_mean_abs_diff": 0.20987968146800995, + "corr_frobenius": 3.224473476409912, + "lag1_corr_mean_abs_diff": 0.2387717217206955, + "lag1_corr_frobenius": 3.581017255783081 + } + } + }, + "frequency": { + "avg_psd_l1": 0.022738220170140266, + "avg_psd_cosine": 0.4610892964421578, + "avg_low_high_ratio_abs_diff": 18.224427510946988 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 308.0, + "mean_dwell": 1.2427184466019416, + "median_dwell": 1.0, + "mean_step": 0.8413416007896523, + "median_step": 0.7274894714355469 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 347.0, + "mean_dwell": 1.103448275862069, + "median_dwell": 1.0, + "mean_step": 41.3141472878254, + "median_step": 54.116028785705566 + }, + "P1_B2004": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 0.014068839402653235, + "median_step": 0.0017450004816055298 + }, + "P1_B3004": { + "num_changes": 370.0, + "mean_dwell": 1.0350404312668464, + "median_dwell": 1.0, + "mean_step": 14.77256890374261, + "median_step": 10.804779052734375 + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.8157603410765646, + "median_step": 0.5944099426269531 + }, + "P1_B3005": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 64.28072175926926, + "median_step": 41.228179931640625 + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22997.85714285714, + "mean_abs_diff_mean_dwell": 14417.281142645239, + "mean_abs_diff_median_dwell": 13988.57142857143, + "mean_abs_diff_mean_step": 19.03385039702909, + "mean_abs_diff_median_step": 16.840154950817425 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6822916666666666, + "change_rate": 0.7702349869451697, + "step_median": 68.81373023986816 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.09333292643229163, + "mean_abs_diff_change_rate": 0.309414869553025, + "mean_abs_diff_step_median": 68.78170993924141 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.015625, + "top1_mass": 0.40625, + "top3_mass": 0.7760416666666666, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8776041666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.059895833333333336, + "top3_mass": 0.0703125, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.18229166666666666, + "top3_mass": 0.3697916666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.8463541666666666, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4817708333333333, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.7864583333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3515625, + "top1_mass": 0.036458333333333336, + "top3_mass": 0.09635416666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5303484598795573, + "mean_abs_diff_top1_mass": 0.03112411499023438, + "mean_abs_diff_top3_mass": 0.05629857381184896, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17409999668598175, + "q50": 0.17409999668598175, + "q95": 0.27557000517845154, + "tail_ratio": NaN + }, + "P2_SIT02": { + "q05": 774.923583984375, + "q50": 784.5628051757812, + "q95": 791.695068359375, + "tail_ratio": 0.7399211037871448 + }, + "P1_FT03": { + "q05": 209.12059020996094, + "q50": 298.9196472167969, + "q95": 316.24639892578125, + "tail_ratio": 0.19295026347175762 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.644094243645668, + "mean_abs_diff_q50": 2.27042847375075, + "mean_abs_diff_q95": 3.5563383996486664, + "mean_abs_diff_tail_ratio": 0.04706495018632424 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 64.74391126130756, + "median_step": 62.410614013671875 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 1312.3134692358592, + "median_step": 1284.6439208984375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.09039718638964156, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 680.877589673068, + "mean_abs_diff_median_step": 670.2603607177734 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 35.254512786865234, + "std": 31.650897979736328, + "lag1": -0.0927804708480835 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.003297113813459873, + "lag1": 0.06643600016832352 + }, + "P2_HILout": { + "mean": 711.4987182617188, + "std": 7.856828212738037, + "lag1": -0.013758387416601181 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.4565467834472656, + "mean_abs_diff_std": 0.2373291968833655, + "mean_abs_diff_lag1": 0.4579590285817782 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9622408151626587, + "mae": 0.7748920321464539 + }, + "real_plus_synthetic": { + "rmse": 0.5504262447357178, + "mae": 0.35703518986701965 + }, + "rmse_ratio_synth_to_real": 1.7233620999417778 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8290584415584414, + "auprc": 0.5923604480636702, + "threshold": 0.006162473000586033, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6756756756756758, + "best_f1_threshold": 1.4353219270706177 + }, + "real_plus_synthetic": { + "auroc": 0.8748840445269016, + "auprc": 0.6416395524092404, + "threshold": 0.30959632992744446, + "f1": 0.25196850393700787, + "precision": 0.1523809523809524, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.552984893321991 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_snr_weight__seed1337/eval.json b/example/results/ablations/runs/no_snr_weight__seed1337/eval.json new file mode 100644 index 0000000..46a8e87 --- /dev/null +++ b/example/results/ablations/runs/no_snr_weight__seed1337/eval.json @@ -0,0 +1,974 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.11384, + "std": 0.0 + }, + "P1_B2016": { + "mean": 1.3915789583333333, + "std": 0.13951615663369016 + }, + "P1_B3004": { + "mean": 396.00211, + "std": 0.0 + }, + "P1_B3005": { + "mean": 1015.43976, + "std": 0.0 + }, + "P1_B4002": { + "mean": 33.54375, + "std": 0.0 + }, + "P1_B4005": { + "mean": 0.0, + "std": 0.0 + }, + "P1_B400B": { + "mean": 1774.7827939583326, + "std": 1196.249609157371 + }, + "P1_B4022": { + "mean": 36.881583541666664, + "std": 0.14105102171131376 + }, + "P1_FCV02Z": { + "mean": 22.649096953125017, + "std": 40.02980408940943 + }, + "P1_FCV03D": { + "mean": 58.57132513020832, + "std": 6.641758179865945 + }, + "P1_FCV03Z": { + "mean": 58.90815119791667, + "std": 7.001653005350173 + }, + "P1_FT01": { + "mean": 194.87539984374996, + "std": 31.95704148207855 + }, + "P1_FT01Z": { + "mean": 877.9929187239587, + "std": 88.39585465010407 + }, + "P1_FT02": { + "mean": 1130.2433795572906, + "std": 890.8762181787059 + }, + "P1_FT02Z": { + "mean": 1130.2433795572906, + "std": 890.8762181787059 + }, + "P1_FT03": { + "mean": 282.5776710677085, + "std": 33.729988496983445 + }, + "P1_FT03Z": { + "mean": 282.5776710677085, + "std": 33.729988496983445 + }, + "P1_LCV01D": { + "mean": 12.167291770833325, + "std": 3.045686552374071 + }, + "P1_LCV01Z": { + "mean": 11.623581380208337, + "std": 3.115988372905861 + }, + "P1_LIT01": { + "mean": 404.0177025000001, + "std": 15.875418171676097 + }, + "P1_PCV01D": { + "mean": 67.45024533854168, + "std": 23.851350116634656 + }, + "P1_PCV01Z": { + "mean": 68.27232518229168, + "std": 23.94305647540201 + }, + "P1_PCV02Z": { + "mean": 11.76605, + "std": 0.0 + }, + "P1_PIT01": { + "mean": 1.3656589583333334, + "std": 0.06896384474826658 + }, + "P1_PIT02": { + "mean": 0.22165216145833339, + "std": 0.050713451923733784 + }, + "P1_TIT01": { + "mean": 35.664832421874976, + "std": 0.581411930313464 + }, + "P1_TIT02": { + "mean": 36.50254908854167, + "std": 1.230454577883268 + }, + "P2_24Vdc": { + "mean": 28.028640781249994, + "std": 0.003504988301149289 + }, + "P2_CO_rpm": { + "mean": 54102.263132760454, + "std": 24.22991470965418 + }, + "P2_HILout": { + "mean": 711.4869008854171, + "std": 9.422904336246441 + }, + "P2_MSD": { + "mean": 758.4425, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.1203645833333, + "std": 5.863175595627757 + }, + "P2_SIT02": { + "mean": 783.9243950520832, + "std": 5.842073451728935 + }, + "P2_VT01": { + "mean": 11.92377559895834, + "std": 0.06899504734301475 + }, + "P2_VXT02": { + "mean": -3.2993418229166647, + "std": 0.47732142243099734 + }, + "P2_VXT03": { + "mean": -1.3056558072916657, + "std": 0.357358577684624 + }, + "P2_VYT02": { + "mean": 3.9809067447916653, + "std": 0.5056739269819277 + }, + "P2_VYT03": { + "mean": 6.233385781250003, + "std": 0.3890750532936693 + }, + "P3_FIT01": { + "mean": 923.9914200520819, + "std": 1652.5150467263375 + }, + "P3_LCP01D": { + "mean": 4590.935969635418, + "std": 5143.60672748293 + }, + "P3_LCV01D": { + "mean": 7112.66516984375, + "std": 6868.979309599681 + }, + "P3_LIT01": { + "mean": 13352.549798333343, + "std": 4371.181160177564 + }, + "P3_PIT01": { + "mean": 349.60746950520803, + "std": 1059.1836216633467 + }, + "P4_HT_FD": { + "mean": -0.0009628385416666669, + "std": 0.007058789792277817 + }, + "P4_HT_LD": { + "mean": -115.69799739583333, + "std": 35.27327640165136 + }, + "P4_HT_PO": { + "mean": 34.00062229166666, + "std": 31.54228311535166 + }, + "P4_LD": { + "mean": 366.12835208333337, + "std": 61.029872497703124 + }, + "P4_ST_FD": { + "mean": -6.520833333333327e-05, + "std": 0.0040470190116635085 + }, + "P4_ST_GOV": { + "mean": 17918.342076744808, + "std": 1896.1799616733456 + }, + "P4_ST_LD": { + "mean": 332.0939757812499, + "std": 38.59835141654271 + }, + "P4_ST_PO": { + "mean": 332.3675919270833, + "std": 35.2570832506201 + }, + "P4_ST_PT01": { + "mean": 10048.308759036465, + "std": 18.72132950187206 + }, + "P4_ST_TT01": { + "mean": 27608.200469973974, + "std": 24.77120613410809 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.020272347909024324, + "std_abs_err": 0.017998437082092405 + }, + "P1_B2016": { + "mean_abs_err": 0.00031263366856637553, + "std_abs_err": 0.003742594157127077 + }, + "P1_B3004": { + "mean_abs_err": 7.383953824526145, + "std_abs_err": 15.41814958804181 + }, + "P1_B3005": { + "mean_abs_err": 45.609026528629784, + "std_abs_err": 61.18331651100028 + }, + "P1_B4002": { + "mean_abs_err": 1.2268236801201198, + "std_abs_err": 0.5786734991990778 + }, + "P1_B4005": { + "mean_abs_err": 60.52510701619792, + "std_abs_err": 42.18129744770969 + }, + "P1_B400B": { + "mean_abs_err": 3.6930762196354863, + "std_abs_err": 2.897336238298294 + }, + "P1_B4022": { + "mean_abs_err": 0.8204088942641405, + "std_abs_err": 0.5303998331235046 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.301837947347259, + "std_abs_err": 0.4086813498487487 + }, + "P1_FCV03D": { + "mean_abs_err": 0.0034616793614929975, + "std_abs_err": 0.04226246115718979 + }, + "P1_FCV03Z": { + "mean_abs_err": 0.39088798227316346, + "std_abs_err": 0.2868490410185709 + }, + "P1_FT01": { + "mean_abs_err": 0.15174548746969663, + "std_abs_err": 4.609059643434225 + }, + "P1_FT01Z": { + "mean_abs_err": 1.0567627801938215, + "std_abs_err": 12.630151686785823 + }, + "P1_FT02": { + "mean_abs_err": 2.5747736773355427, + "std_abs_err": 1.6800543272933055 + }, + "P1_FT02Z": { + "mean_abs_err": 648.2350670513581, + "std_abs_err": 302.46913759244364 + }, + "P1_FT03": { + "mean_abs_err": 0.06129920852043824, + "std_abs_err": 0.2557753856879401 + }, + "P1_FT03Z": { + "mean_abs_err": 778.4643672808131, + "std_abs_err": 27.395500366633883 + }, + "P1_LCV01D": { + "mean_abs_err": 0.012959109058188645, + "std_abs_err": 0.13107159934394952 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.34249264912958033, + "std_abs_err": 0.15251887556054955 + }, + "P1_LIT01": { + "mean_abs_err": 0.09716966300618424, + "std_abs_err": 0.24352012330212602 + }, + "P1_PCV01D": { + "mean_abs_err": 0.0153283971497018, + "std_abs_err": 0.08158341572908157 + }, + "P1_PCV01Z": { + "mean_abs_err": 0.13582661461090595, + "std_abs_err": 0.07845666473836488 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.07670294660492694, + "std_abs_err": 0.09684446546464737 + }, + "P1_PIT01": { + "mean_abs_err": 0.011822262420058571, + "std_abs_err": 0.007531285378384482 + }, + "P1_PIT02": { + "mean_abs_err": 0.26550492888318333, + "std_abs_err": 0.44799818613114606 + }, + "P1_TIT01": { + "mean_abs_err": 0.016375820558238274, + "std_abs_err": 0.0018686968724849207 + }, + "P1_TIT02": { + "mean_abs_err": 0.02242958979486076, + "std_abs_err": 0.00869585185900501 + }, + "P2_24Vdc": { + "mean_abs_err": 6.8921377675224e-05, + "std_abs_err": 0.00016252218674237418 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.44706512222182937, + "std_abs_err": 1.7741829348658165 + }, + "P2_HILout": { + "mean_abs_err": 0.02409161532091275, + "std_abs_err": 0.9681784589902787 + }, + "P2_MSD": { + "mean_abs_err": 4.750739999999951, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.7835528301096701, + "std_abs_err": 0.6642602351835318 + }, + "P2_SIT02": { + "mean_abs_err": 0.02065716953700303, + "std_abs_err": 0.6514936949159686 + }, + "P2_VT01": { + "mean_abs_err": 4.00470517938345e-05, + "std_abs_err": 0.000481365271012682 + }, + "P2_VXT02": { + "mean_abs_err": 0.0008366494733316188, + "std_abs_err": 0.006288496159777057 + }, + "P2_VXT03": { + "mean_abs_err": 0.001237479373012107, + "std_abs_err": 0.007731198520970317 + }, + "P2_VYT02": { + "mean_abs_err": 0.0005621052012716454, + "std_abs_err": 0.0057634739891398445 + }, + "P2_VYT03": { + "mean_abs_err": 0.0010193315478819187, + "std_abs_err": 0.009272091322577725 + }, + "P3_FIT01": { + "mean_abs_err": 226.90875605410122, + "std_abs_err": 127.20965106393737 + }, + "P3_LCP01D": { + "mean_abs_err": 6.208821394843653, + "std_abs_err": 13.606729767051547 + }, + "P3_LCV01D": { + "mean_abs_err": 21.211547137491834, + "std_abs_err": 29.05141378385997 + }, + "P3_LIT01": { + "mean_abs_err": 2.2946943606439163, + "std_abs_err": 17.006523249628117 + }, + "P3_PIT01": { + "mean_abs_err": 299.7504335181197, + "std_abs_err": 91.73830904384863 + }, + "P4_HT_FD": { + "mean_abs_err": 0.000859650596314927, + "std_abs_err": 0.005026771477873717 + }, + "P4_HT_LD": { + "mean_abs_err": 149.69291447551893, + "std_abs_err": 1.8494828916530537 + }, + "P4_HT_PO": { + "mean_abs_err": 0.016814496909752563, + "std_abs_err": 0.10141189803018236 + }, + "P4_LD": { + "mean_abs_err": 0.00040171086095597275, + "std_abs_err": 0.41406612826011724 + }, + "P4_ST_FD": { + "mean_abs_err": 3.609510345018445e-05, + "std_abs_err": 0.0024048427536734794 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.380815497283038, + "std_abs_err": 44.036745685572214 + }, + "P4_ST_LD": { + "mean_abs_err": 0.09107861143223772, + "std_abs_err": 0.7451821249251651 + }, + "P4_ST_PO": { + "mean_abs_err": 0.0986206034245356, + "std_abs_err": 0.8061397482117982 + }, + "P4_ST_PT01": { + "mean_abs_err": 2.266397782617787, + "std_abs_err": 0.9365594765529295 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.8322217055574583, + "std_abs_err": 0.8538843295397633 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 1.0, + "P1_B2016": 0.04309583333333333, + "P1_B3004": 0.8108, + "P1_B3005": 0.8106, + "P1_B4002": 0.81055, + "P1_B4005": 0.9113, + "P1_B400B": 0.4580666666666666, + "P1_B4022": 0.5730333333333334, + "P1_FCV02Z": 0.6847041666666667, + "P1_FCV03D": 0.4880875, + "P1_FCV03Z": 0.4260833333333333, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4746, + "P1_FT02": 0.45826666666666666, + "P1_FT02Z": 0.8361000000000001, + "P1_FT03": 0.5241, + "P1_FT03Z": 1.0, + "P1_LCV01D": 0.44748750000000004, + "P1_LCV01Z": 0.4546666666666667, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5573083333333333, + "P1_PCV01Z": 0.5588041666666668, + "P1_PCV02Z": 1.0, + "P1_PIT01": 0.15849999999999997, + "P1_PIT02": 0.61435, + "P1_TIT01": 0.09710416666666666, + "P1_TIT02": 0.09962083333333338, + "P2_24Vdc": 0.12001666666666666, + "P2_CO_rpm": 0.09774166666666667, + "P2_HILout": 0.08204583333333332, + "P2_MSD": 1.0, + "P2_SIT01": 0.4593125, + "P2_SIT02": 0.5010625, + "P2_VT01": 0.1226708333333334, + "P2_VXT02": 0.2817291666666666, + "P2_VXT03": 0.43657500000000005, + "P2_VYT02": 0.25022083333333334, + "P2_VYT03": 0.2700083333333334, + "P3_FIT01": 0.2109375, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.0937541666666667, + "P3_LIT01": 0.09344166666666665, + "P3_PIT01": 0.3359375, + "P4_HT_FD": 0.16174583333333337, + "P4_HT_LD": 0.9375, + "P4_HT_PO": 0.0811416666666667, + "P4_LD": 0.03089583333333329, + "P4_ST_FD": 0.023541666666666683, + "P4_ST_GOV": 0.035420833333333346, + "P4_ST_LD": 0.032491666666666585, + "P4_ST_PO": 0.035720833333333424, + "P4_ST_PT01": 0.1282875, + "P4_ST_TT01": 0.07099583333333337 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.08333, + "q25_diff": 0.012630000000000002, + "q50_diff": 0.012630000000000002, + "q75_diff": 0.012630000000000002, + "q95_diff": 0.012630000000000002 + }, + "P1_B2016": { + "q05_diff": 0.01142999999999983, + "q25_diff": 0.010210000000000052, + "q50_diff": 0.005260000000000042, + "q75_diff": 0.01419999999999999, + "q95_diff": 0.03000999999999987 + }, + "P1_B3004": { + "q05_diff": 2.8398500000000126, + "q25_diff": 1.6357399999999984, + "q50_diff": 1.6357399999999984, + "q75_diff": 1.6357399999999984, + "q95_diff": 1.6357399999999984 + }, + "P1_B3005": { + "q05_diff": 13.44177000000002, + "q25_diff": 13.44177000000002, + "q50_diff": 13.44177000000002, + "q75_diff": 13.44177000000002, + "q95_diff": 99.67999000000009 + }, + "P1_B4002": { + "q05_diff": 1.5437500000000028, + "q25_diff": 0.11175000000000068, + "q50_diff": 0.11175000000000068, + "q75_diff": 0.11175000000000068, + "q95_diff": 0.11175000000000068 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 100.0, + "q50_diff": 100.0, + "q75_diff": 100.0, + "q95_diff": 100.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2493.20291, + "q50_diff": 193.92164000000002, + "q75_diff": 9.545890000000327, + "q95_diff": 7.019040000000132 + }, + "P1_B4022": { + "q05_diff": 1.2294699999999992, + "q25_diff": 0.1574500000000043, + "q50_diff": 0.4875299999999996, + "q75_diff": 0.5946499999999943, + "q95_diff": 0.6461800000000011 + }, + "P1_FCV02Z": { + "q05_diff": 0.015249999999999986, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.015260000000000051, + "q75_diff": 33.11158, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 2.895609999999998, + "q25_diff": 0.09575999999999851, + "q50_diff": 8.542679999999997, + "q75_diff": 11.14949, + "q95_diff": 0.7052599999999956 + }, + "P1_FCV03Z": { + "q05_diff": 3.52478, + "q25_diff": 0.01525999999999783, + "q50_diff": 8.96454, + "q75_diff": 11.100770000000004, + "q95_diff": 0.5172800000000137 + }, + "P1_FT01": { + "q05_diff": 7.965710000000001, + "q25_diff": 8.585710000000006, + "q50_diff": 32.806329999999974, + "q75_diff": 37.76549, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 39.49407999999994, + "q25_diff": 17.395080000000007, + "q50_diff": 73.92865000000006, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.3297499999999, + "q50_diff": 334.3580300000001, + "q75_diff": 12.970209999999952, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 27.82217, + "q25_diff": 2761.27054, + "q50_diff": 1193.93238, + "q75_diff": 870.8322700000001, + "q95_diff": 865.2337600000001 + }, + "P1_FT03": { + "q05_diff": 35.983990000000006, + "q25_diff": 1.5271900000000187, + "q50_diff": 51.52633000000003, + "q75_diff": 59.781600000000026, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 788.71785, + "q25_diff": 752.67152, + "q50_diff": 703.89301, + "q75_diff": 697.59076, + "q95_diff": 800.17028 + }, + "P1_LCV01D": { + "q05_diff": 0.10884999999999945, + "q25_diff": 1.2068000000000012, + "q50_diff": 4.63546, + "q75_diff": 3.3100299999999994, + "q95_diff": 0.6150000000000002 + }, + "P1_LCV01Z": { + "q05_diff": 0.4653900000000002, + "q25_diff": 0.3738399999999995, + "q50_diff": 3.8070700000000013, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.7163599999999999 + }, + "P1_LIT01": { + "q05_diff": 14.272339999999986, + "q25_diff": 1.0244100000000458, + "q50_diff": 6.281960000000026, + "q75_diff": 8.229829999999993, + "q95_diff": 34.75434000000001 + }, + "P1_PCV01D": { + "q05_diff": 1.0878000000000014, + "q25_diff": 3.7107699999999966, + "q50_diff": 22.24723999999999, + "q75_diff": 44.833270000000006, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.9393799999999999, + "q25_diff": 3.814699999999995, + "q50_diff": 22.722680000000004, + "q75_diff": 45.46356000000001, + "q95_diff": 45.14488 + }, + "P1_PCV02Z": { + "q05_diff": 0.2441399999999998, + "q25_diff": 0.2441399999999998, + "q50_diff": 0.2517700000000005, + "q75_diff": 0.2517700000000005, + "q95_diff": 0.2517700000000005 + }, + "P1_PIT01": { + "q05_diff": 0.020600000000000174, + "q25_diff": 0.006870000000000154, + "q50_diff": 0.012210000000000054, + "q75_diff": 0.02364999999999995, + "q95_diff": 0.04959000000000002 + }, + "P1_PIT02": { + "q05_diff": 0.03051999999999999, + "q25_diff": 0.10146999999999998, + "q50_diff": 0.10223999999999997, + "q75_diff": 0.0015300000000000313, + "q95_diff": 0.0022900000000000142 + }, + "P1_TIT01": { + "q05_diff": 0.06425000000000125, + "q25_diff": 0.07629999999999626, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.22887999999999664, + "q95_diff": 0.07629000000000019 + }, + "P1_TIT02": { + "q05_diff": 0.0, + "q25_diff": 0.01525999999999783, + "q50_diff": 0.13732999999999862, + "q75_diff": 0.25939000000000334, + "q95_diff": 0.4425099999999986 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 7.000000000090267e-05, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.000920000000000698 + }, + "P2_CO_rpm": { + "q05_diff": 11.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 2.0, + "q95_diff": 0.2656199999983073 + }, + "P2_HILout": { + "q05_diff": 0.5208799999999201, + "q25_diff": 2.0201399999999694, + "q50_diff": 0.08233999999993102, + "q75_diff": 1.539190000000076, + "q95_diff": 1.1491100000000642 + }, + "P2_MSD": { + "q05_diff": 4.750739999999951, + "q25_diff": 4.750739999999951, + "q50_diff": 4.750739999999951, + "q75_diff": 4.750739999999951, + "q95_diff": 4.750739999999951 + }, + "P2_SIT01": { + "q05_diff": 3.0, + "q25_diff": 2.0, + "q50_diff": 4.0, + "q75_diff": 6.0, + "q95_diff": 7.0 + }, + "P2_SIT02": { + "q05_diff": 2.6633900000000494, + "q25_diff": 3.5723900000000413, + "q50_diff": 5.610480000000052, + "q75_diff": 6.367129999999975, + "q95_diff": 6.833739999999921 + }, + "P2_VT01": { + "q05_diff": 0.00835000000000008, + "q25_diff": 0.015249999999999986, + "q50_diff": 0.014089999999999492, + "q75_diff": 0.013489999999999114, + "q95_diff": 0.018819999999999837 + }, + "P2_VXT02": { + "q05_diff": 0.13024000000000058, + "q25_diff": 0.29942, + "q50_diff": 0.28566, + "q75_diff": 0.26685000000000025, + "q95_diff": 0.45981000000000005 + }, + "P2_VXT03": { + "q05_diff": 0.10094999999999987, + "q25_diff": 0.26912999999999987, + "q50_diff": 0.29733999999999994, + "q75_diff": 0.32732000000000006, + "q95_diff": 0.61116 + }, + "P2_VYT02": { + "q05_diff": 0.09450000000000003, + "q25_diff": 0.26248000000000005, + "q50_diff": 0.22676000000000007, + "q75_diff": 0.2350399999999997, + "q95_diff": 0.38853000000000026 + }, + "P2_VYT03": { + "q05_diff": 0.05539999999999967, + "q25_diff": 0.18588000000000005, + "q50_diff": 0.17625000000000046, + "q75_diff": 0.18273000000000028, + "q95_diff": 0.35335999999999945 + }, + "P3_FIT01": { + "q05_diff": 3.0, + "q25_diff": 0.0, + "q50_diff": 70.0, + "q75_diff": 98.0, + "q95_diff": 114.0 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 103.36768000000006, + "q75_diff": 127.4433599999993, + "q95_diff": 11.193359999999302 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 256.0, + "q50_diff": 3824.0, + "q75_diff": 161.67871000000014, + "q95_diff": 31.040039999999863 + }, + "P3_LIT01": { + "q05_diff": 1134.7026400000004, + "q25_diff": 27.31151999999929, + "q50_diff": 650.7148400000005, + "q75_diff": 1140.0781199999983, + "q95_diff": 1.0507800000013958 + }, + "P3_PIT01": { + "q05_diff": 4.0, + "q25_diff": 5.0, + "q50_diff": 0.0, + "q75_diff": 928.0, + "q95_diff": 62.24805000000015 + }, + "P4_HT_FD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 7.000000000000002e-05, + "q75_diff": 7e-05, + "q95_diff": 0.0 + }, + "P4_HT_LD": { + "q05_diff": 124.7862, + "q25_diff": 124.7862, + "q50_diff": 179.53157, + "q75_diff": 193.36982, + "q95_diff": 56.57957999999999 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 1.24783, + "q50_diff": 14.768200000000004, + "q75_diff": 0.7967900000000014, + "q95_diff": 0.1009300000000053 + }, + "P4_LD": { + "q05_diff": 0.43831999999997606, + "q25_diff": 3.1083299999999667, + "q50_diff": 3.4359999999999786, + "q75_diff": 0.03201000000001386, + "q95_diff": 4.201050000000009 + }, + "P4_ST_FD": { + "q05_diff": 3.999999999999967e-05, + "q25_diff": 0.00013999999999999993, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 8.000000000000021e-05 + }, + "P4_ST_GOV": { + "q05_diff": 116.28319999999985, + "q25_diff": 38.044920000000275, + "q50_diff": 31.888670000000275, + "q75_diff": 110.5820300000014, + "q95_diff": 413.2324200000003 + }, + "P4_ST_LD": { + "q05_diff": 0.7356900000000337, + "q25_diff": 0.4004799999999591, + "q50_diff": 0.47867000000002236, + "q75_diff": 2.0575600000000236, + "q95_diff": 7.868069999999989 + }, + "P4_ST_PO": { + "q05_diff": 2.1548199999999724, + "q25_diff": 0.7244600000000219, + "q50_diff": 0.6116099999999847, + "q75_diff": 2.1616500000000087, + "q95_diff": 7.556669999999997 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 20.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 6.0, + "q50_diff": 3.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0189476811306442e-05, + "P1_B2016": 0.06955732600083286, + "P1_B3004": 1.018740504876714e-05, + "P1_B3005": 0.9999898125942444, + "P1_B4002": 0.9999898146591784, + "P1_B4005": 0.9999997508214005, + "P1_B400B": 0.5854431525410038, + "P1_B4022": 0.03037469907643997, + "P1_FCV02Z": 0.06392491640099707, + "P1_FCV03D": 0.2645428296063813, + "P1_FCV03Z": 0.14097187280714174, + "P1_FT01": 0.47270955964658845, + "P1_FT01Z": 0.36815633159095196, + "P1_FT02": 0.7916960456221444, + "P1_FT02Z": 0.7917510242320559, + "P1_FT03": 0.5895476319903267, + "P1_FT03Z": 0.5894105749457198, + "P1_LCV01D": 0.17282397457797494, + "P1_LCV01Z": 0.21892854220313507, + "P1_LIT01": 0.11337733428660046, + "P1_PCV01D": 0.1803572897957656, + "P1_PCV01Z": 0.09548514944092523, + "P1_PCV02Z": 0.5669400491575541, + "P1_PIT01": 0.1875869792291166, + "P1_PIT02": 0.19751880548222533, + "P1_TIT01": 0.04303664560561393, + "P1_TIT02": 0.08718233760664018, + "P2_24Vdc": 0.15235368502801752, + "P2_CO_rpm": 0.42742755199484195, + "P2_HILout": 0.2042824724688297, + "P2_MSD": 0.0, + "P2_SIT01": 0.43609660500873776, + "P2_SIT02": 0.5838584105971032, + "P2_VT01": 0.1492203579005219, + "P2_VXT02": 0.16161782957088977, + "P2_VXT03": 0.17996575274100857, + "P2_VYT02": 0.1464148112045175, + "P2_VYT03": 0.16868545803734847, + "P3_FIT01": 0.18381695503960072, + "P3_LCP01D": 0.20447780712073593, + "P3_LCV01D": 0.27862298184654877, + "P3_LIT01": 0.06847572452877615, + "P3_PIT01": 0.8461534722886025, + "P4_HT_FD": 0.03635399688775193, + "P4_HT_LD": 0.15670838558697175, + "P4_HT_PO": 0.4151919928262455, + "P4_LD": 0.06943187697091346, + "P4_ST_FD": 0.28469546148549274, + "P4_ST_GOV": 0.0913873968521397, + "P4_ST_LD": 0.13676128749029393, + "P4_ST_PO": 0.08652073760594847, + "P4_ST_PT01": 0.6715692188345836, + "P4_ST_TT01": 0.1081291586433758 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.3825871639778149, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + }, + "avg_ks": 0.39950668238993714, + "avg_jsd": 0.02154500462890076, + "avg_lag1_diff": 0.2994253248181626 +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_snr_weight__seed1337/eval_post.json b/example/results/ablations/runs/no_snr_weight__seed1337/eval_post.json new file mode 100644 index 0000000..11e6ac2 --- /dev/null +++ b/example/results/ablations/runs/no_snr_weight__seed1337/eval_post.json @@ -0,0 +1,971 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09355486979166669, + "std": 0.01799194293084728 + }, + "P1_B2016": { + "mean": 1.3915789583333333, + "std": 0.13951615663369016 + }, + "P1_B3004": { + "mean": 403.2905014583335, + "std": 14.217563527951576 + }, + "P1_B3005": { + "mean": 1059.5253898958326, + "std": 58.42587703751758 + }, + "P1_B4002": { + "mean": 32.37490617187499, + "std": 0.6519533213854061 + }, + "P1_B4005": { + "mean": 64.99571291666655, + "std": 42.03467331899641 + }, + "P1_B400B": { + "mean": 1774.7827939583326, + "std": 1196.249609157371 + }, + "P1_B4022": { + "mean": 36.08671476562498, + "std": 0.7147782383668128 + }, + "P1_FCV02Z": { + "mean": 21.954832630208315, + "std": 38.87335152465215 + }, + "P1_FCV03D": { + "mean": 58.14541601562505, + "std": 6.267267768996455 + }, + "P1_FCV03Z": { + "mean": 58.21398343750003, + "std": 6.423211358701469 + }, + "P1_FT01": { + "mean": 194.87539984374996, + "std": 31.95704148207855 + }, + "P1_FT01Z": { + "mean": 877.9929187239587, + "std": 88.39585465010407 + }, + "P1_FT02": { + "mean": 1130.2433795572906, + "std": 890.8762181787059 + }, + "P1_FT02Z": { + "mean": 1843.7261292187509, + "std": 1190.4241853713818 + }, + "P1_FT03": { + "mean": 282.5776710677085, + "std": 33.729988496983445 + }, + "P1_FT03Z": { + "mean": 1058.2564329947922, + "std": 59.50957255113764 + }, + "P1_LCV01D": { + "mean": 11.699055963541667, + "std": 3.0155193220433567 + }, + "P1_LCV01Z": { + "mean": 11.627510442708333, + "std": 3.0251072332459787 + }, + "P1_LIT01": { + "mean": 404.0177025000001, + "std": 15.875418171676097 + }, + "P1_PCV01D": { + "mean": 65.64593611979173, + "std": 23.56629129628467 + }, + "P1_PCV01Z": { + "mean": 65.565628671875, + "std": 23.790956943468952 + }, + "P1_PCV02Z": { + "mean": 11.867279765625007, + "std": 0.10811550492611105 + }, + "P1_PIT01": { + "mean": 1.3656589583333334, + "std": 0.06896384474826658 + }, + "P1_PIT02": { + "mean": 0.22165216145833339, + "std": 0.050713451923733784 + }, + "P1_TIT01": { + "mean": 35.664832421874976, + "std": 0.581411930313464 + }, + "P1_TIT02": { + "mean": 36.50254908854167, + "std": 1.230454577883268 + }, + "P2_24Vdc": { + "mean": 28.02860809895833, + "std": 0.0032970984602375326 + }, + "P2_CO_rpm": { + "mean": 54102.263132760454, + "std": 24.22991470965418 + }, + "P2_HILout": { + "mean": 711.4986896354167, + "std": 7.856828438379631 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.1203645833333, + "std": 5.863175595627757 + }, + "P2_SIT02": { + "mean": 783.9243950520832, + "std": 5.842073451728935 + }, + "P2_VT01": { + "mean": 11.92377559895834, + "std": 0.06899504734301475 + }, + "P2_VXT02": { + "mean": -3.2993418229166647, + "std": 0.47732142243099734 + }, + "P2_VXT03": { + "mean": -1.3056558072916657, + "std": 0.357358577684624 + }, + "P2_VYT02": { + "mean": 3.9809067447916653, + "std": 0.5056739269819277 + }, + "P2_VYT03": { + "mean": 6.233385781250003, + "std": 0.3890750532936693 + }, + "P3_FIT01": { + "mean": 923.9914200520819, + "std": 1652.5150467263375 + }, + "P3_LCP01D": { + "mean": 4590.935969635418, + "std": 5143.60672748293 + }, + "P3_LCV01D": { + "mean": 7112.66516984375, + "std": 6868.979309599681 + }, + "P3_LIT01": { + "mean": 13352.549798333343, + "std": 4371.181160177564 + }, + "P3_PIT01": { + "mean": 349.60746950520803, + "std": 1059.1836216633467 + }, + "P4_HT_FD": { + "mean": -0.0009628385416666669, + "std": 0.007058789792277817 + }, + "P4_HT_LD": { + "mean": 34.231568880208286, + "std": 33.60361854009684 + }, + "P4_HT_PO": { + "mean": 35.25451283854166, + "std": 31.650898099038645 + }, + "P4_LD": { + "mean": 366.12835208333337, + "std": 61.029872497703124 + }, + "P4_ST_FD": { + "mean": -6.520833333333327e-05, + "std": 0.0040470190116635085 + }, + "P4_ST_GOV": { + "mean": 17918.342076744808, + "std": 1896.1799616733456 + }, + "P4_ST_LD": { + "mean": 332.0939757812499, + "std": 38.59835141654271 + }, + "P4_ST_PO": { + "mean": 332.3675919270833, + "std": 35.2570832506201 + }, + "P4_ST_PT01": { + "mean": 10048.308759036465, + "std": 18.72132950187206 + }, + "P4_ST_TT01": { + "mean": 27608.200469973974, + "std": 24.77120613410809 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 1.2782299308983225e-05, + "std_abs_err": 6.494151245123314e-06 + }, + "P1_B2016": { + "mean_abs_err": 0.00031263366856637553, + "std_abs_err": 0.003742594157127077 + }, + "P1_B3004": { + "mean_abs_err": 0.09556236619266656, + "std_abs_err": 1.2005860600902345 + }, + "P1_B3005": { + "mean_abs_err": 1.5233966327971302, + "std_abs_err": 2.757439473482698 + }, + "P1_B4002": { + "mean_abs_err": 0.05797985199510691, + "std_abs_err": 0.07327982218632823 + }, + "P1_B4005": { + "mean_abs_err": 4.470605900468627, + "std_abs_err": 0.14662412871327746 + }, + "P1_B400B": { + "mean_abs_err": 3.6930762196354863, + "std_abs_err": 2.897336238298294 + }, + "P1_B4022": { + "mean_abs_err": 0.02554011822245883, + "std_abs_err": 0.04332738353199439 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.996102270263961, + "std_abs_err": 1.5651339146060295 + }, + "P1_FCV03D": { + "mean_abs_err": 0.42244743522177686, + "std_abs_err": 0.3322279497122995 + }, + "P1_FCV03Z": { + "mean_abs_err": 1.0850557426898035, + "std_abs_err": 0.29159260563013323 + }, + "P1_FT01": { + "mean_abs_err": 0.15174548746969663, + "std_abs_err": 4.609059643434225 + }, + "P1_FT01Z": { + "mean_abs_err": 1.0567627801938215, + "std_abs_err": 12.630151686785823 + }, + "P1_FT02": { + "mean_abs_err": 2.5747736773355427, + "std_abs_err": 1.6800543272933055 + }, + "P1_FT02Z": { + "mean_abs_err": 65.24768261010217, + "std_abs_err": 2.9211703997677887 + }, + "P1_FT03": { + "mean_abs_err": 0.06129920852043824, + "std_abs_err": 0.2557753856879401 + }, + "P1_FT03Z": { + "mean_abs_err": 2.785605353729352, + "std_abs_err": 1.6159163124796905 + }, + "P1_LCV01D": { + "mean_abs_err": 0.45527669823346884, + "std_abs_err": 0.1009043690132354 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.33856358662958463, + "std_abs_err": 0.061637735900667234 + }, + "P1_LIT01": { + "mean_abs_err": 0.09716966300618424, + "std_abs_err": 0.24352012330212602 + }, + "P1_PCV01D": { + "mean_abs_err": 1.8196376158996515, + "std_abs_err": 0.20347540462090308 + }, + "P1_PCV01Z": { + "mean_abs_err": 2.842523125027583, + "std_abs_err": 0.07364286719469249 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.024526819020080026, + "std_abs_err": 0.011271039461463683 + }, + "P1_PIT01": { + "mean_abs_err": 0.011822262420058571, + "std_abs_err": 0.007531285378384482 + }, + "P1_PIT02": { + "mean_abs_err": 0.26550492888318333, + "std_abs_err": 0.44799818613114606 + }, + "P1_TIT01": { + "mean_abs_err": 0.016375820558238274, + "std_abs_err": 0.0018686968724849207 + }, + "P1_TIT02": { + "mean_abs_err": 0.02242958979486076, + "std_abs_err": 0.00869585185900501 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00010160366933931186, + "std_abs_err": 4.536765416938244e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.44706512222182937, + "std_abs_err": 1.7741829348658165 + }, + "P2_HILout": { + "mean_abs_err": 0.035880365320508645, + "std_abs_err": 0.5978974388765312 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.7835528301096701, + "std_abs_err": 0.6642602351835318 + }, + "P2_SIT02": { + "mean_abs_err": 0.02065716953700303, + "std_abs_err": 0.6514936949159686 + }, + "P2_VT01": { + "mean_abs_err": 4.00470517938345e-05, + "std_abs_err": 0.000481365271012682 + }, + "P2_VXT02": { + "mean_abs_err": 0.0008366494733316188, + "std_abs_err": 0.006288496159777057 + }, + "P2_VXT03": { + "mean_abs_err": 0.001237479373012107, + "std_abs_err": 0.007731198520970317 + }, + "P2_VYT02": { + "mean_abs_err": 0.0005621052012716454, + "std_abs_err": 0.0057634739891398445 + }, + "P2_VYT03": { + "mean_abs_err": 0.0010193315478819187, + "std_abs_err": 0.009272091322577725 + }, + "P3_FIT01": { + "mean_abs_err": 226.90875605410122, + "std_abs_err": 127.20965106393737 + }, + "P3_LCP01D": { + "mean_abs_err": 6.208821394843653, + "std_abs_err": 13.606729767051547 + }, + "P3_LCV01D": { + "mean_abs_err": 21.211547137491834, + "std_abs_err": 29.05141378385997 + }, + "P3_LIT01": { + "mean_abs_err": 2.2946943606439163, + "std_abs_err": 17.006523249628117 + }, + "P3_PIT01": { + "mean_abs_err": 299.7504335181197, + "std_abs_err": 91.73830904384863 + }, + "P4_HT_FD": { + "mean_abs_err": 0.000859650596314927, + "std_abs_err": 0.005026771477873717 + }, + "P4_HT_LD": { + "mean_abs_err": 0.23665180052267232, + "std_abs_err": 0.17982503009853446 + }, + "P4_HT_PO": { + "mean_abs_err": 1.2707050437847514, + "std_abs_err": 0.21002688171716954 + }, + "P4_LD": { + "mean_abs_err": 0.00040171086095597275, + "std_abs_err": 0.41406612826011724 + }, + "P4_ST_FD": { + "mean_abs_err": 3.609510345018445e-05, + "std_abs_err": 0.0024048427536734794 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.380815497283038, + "std_abs_err": 44.036745685572214 + }, + "P4_ST_LD": { + "mean_abs_err": 0.09107861143223772, + "std_abs_err": 0.7451821249251651 + }, + "P4_ST_PO": { + "mean_abs_err": 0.0986206034245356, + "std_abs_err": 0.8061397482117982 + }, + "P4_ST_PT01": { + "mean_abs_err": 2.266397782617787, + "std_abs_err": 0.9365594765529295 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.8322217055574583, + "std_abs_err": 0.8538843295397633 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.6595583333333334, + "P1_B2016": 0.04309583333333333, + "P1_B3004": 0.671875, + "P1_B3005": 0.5293, + "P1_B4002": 0.7923208333333334, + "P1_B4005": 0.3062625, + "P1_B400B": 0.4580666666666666, + "P1_B4022": 0.5756333333333334, + "P1_FCV02Z": 0.7746, + "P1_FCV03D": 0.45162500000000005, + "P1_FCV03Z": 0.40985833333333327, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4746, + "P1_FT02": 0.45826666666666666, + "P1_FT02Z": 0.4029958333333334, + "P1_FT03": 0.5241, + "P1_FT03Z": 0.46950833333333336, + "P1_LCV01D": 0.3859916666666667, + "P1_LCV01Z": 0.397625, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5304583333333333, + "P1_PCV01Z": 0.5184416666666667, + "P1_PCV02Z": 0.6588541666666666, + "P1_PIT01": 0.15849999999999997, + "P1_PIT02": 0.61435, + "P1_TIT01": 0.09710416666666666, + "P1_TIT02": 0.09962083333333338, + "P2_24Vdc": 0.13548333333333334, + "P2_CO_rpm": 0.09774166666666667, + "P2_HILout": 0.08242499999999997, + "P2_MSD": 0.0, + "P2_SIT01": 0.4593125, + "P2_SIT02": 0.5010625, + "P2_VT01": 0.1226708333333334, + "P2_VXT02": 0.2817291666666666, + "P2_VXT03": 0.43657500000000005, + "P2_VYT02": 0.25022083333333334, + "P2_VYT03": 0.2700083333333334, + "P3_FIT01": 0.2109375, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.0937541666666667, + "P3_LIT01": 0.09344166666666665, + "P3_PIT01": 0.3359375, + "P4_HT_FD": 0.16174583333333337, + "P4_HT_LD": 0.06742500000000001, + "P4_HT_PO": 0.05970416666666667, + "P4_LD": 0.03089583333333329, + "P4_ST_FD": 0.023541666666666683, + "P4_ST_GOV": 0.035420833333333346, + "P4_ST_LD": 0.032491666666666585, + "P4_ST_PO": 0.035720833333333424, + "P4_ST_PT01": 0.1282875, + "P4_ST_TT01": 0.07099583333333337 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.02845, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0011099999999999999, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.01142999999999983, + "q25_diff": 0.010210000000000052, + "q50_diff": 0.005260000000000042, + "q75_diff": 0.01419999999999999, + "q95_diff": 0.03000999999999987 + }, + "P1_B3004": { + "q05_diff": 14.80313000000001, + "q25_diff": 1.6197799999999916, + "q50_diff": 6.726189999999974, + "q75_diff": 8.508819999999957, + "q95_diff": 37.86444 + }, + "P1_B3005": { + "q05_diff": 40.40789999999993, + "q25_diff": 8.045049999999947, + "q50_diff": 89.51202, + "q75_diff": 104.97393, + "q95_diff": 4.4593499999998585 + }, + "P1_B4002": { + "q05_diff": 0.4564699999999995, + "q25_diff": 1.6555000000000035, + "q50_diff": 1.6555000000000035, + "q75_diff": 0.9287000000000063, + "q95_diff": 0.16969000000000278 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 81.96193, + "q50_diff": 0.04000000000000625, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2493.20291, + "q50_diff": 193.92164000000002, + "q75_diff": 9.545890000000327, + "q95_diff": 7.019040000000132 + }, + "P1_B4022": { + "q05_diff": 0.38222999999999985, + "q25_diff": 1.3479000000000028, + "q50_diff": 1.4273799999999994, + "q75_diff": 0.9127399999999994, + "q95_diff": 0.5270299999999963 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.05340999999999996, + "q75_diff": 38.169869999999996, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 1.3654200000000003, + "q25_diff": 0.2554099999999977, + "q50_diff": 8.60333, + "q75_diff": 10.84863, + "q95_diff": 0.824300000000008 + }, + "P1_FCV03Z": { + "q05_diff": 1.7547599999999974, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.148199999999996, + "q75_diff": 10.864249999999998, + "q95_diff": 0.9307899999999876 + }, + "P1_FT01": { + "q05_diff": 7.965710000000001, + "q25_diff": 8.585710000000006, + "q50_diff": 32.806329999999974, + "q75_diff": 37.76549, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 39.49407999999994, + "q25_diff": 17.395080000000007, + "q50_diff": 73.92865000000006, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.3297499999999, + "q50_diff": 334.3580300000001, + "q75_diff": 12.970209999999952, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 1.9862300000000026, + "q25_diff": 2300.54303, + "q50_diff": 14.318849999999657, + "q75_diff": 7.299800000000232, + "q95_diff": 6.317140000000109 + }, + "P1_FT03": { + "q05_diff": 35.983990000000006, + "q25_diff": 1.5271900000000187, + "q50_diff": 51.52633000000003, + "q75_diff": 59.781600000000026, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 23.41980000000001, + "q25_diff": 2.9805299999999306, + "q50_diff": 84.98266000000012, + "q75_diff": 100.6383699999999, + "q95_diff": 3.5079399999999623 + }, + "P1_LCV01D": { + "q05_diff": 0.17497000000000007, + "q25_diff": 0.29953000000000074, + "q50_diff": 4.171619999999999, + "q75_diff": 3.162939999999999, + "q95_diff": 0.7125800000000009 + }, + "P1_LCV01Z": { + "q05_diff": 0.0, + "q25_diff": 0.5874600000000001, + "q50_diff": 4.638670000000001, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.9536700000000007 + }, + "P1_LIT01": { + "q05_diff": 14.272339999999986, + "q25_diff": 1.0244100000000458, + "q50_diff": 6.281960000000026, + "q75_diff": 8.229829999999993, + "q95_diff": 34.75434000000001 + }, + "P1_PCV01D": { + "q05_diff": 0.18625000000000114, + "q25_diff": 2.7490099999999984, + "q50_diff": 18.274499999999996, + "q75_diff": 43.266600000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.22888000000000375, + "q25_diff": 2.0370500000000007, + "q50_diff": 16.029360000000004, + "q75_diff": 43.33495, + "q95_diff": 45.1493 + }, + "P1_PCV02Z": { + "q05_diff": 0.2288800000000002, + "q25_diff": 0.2212499999999995, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_PIT01": { + "q05_diff": 0.020600000000000174, + "q25_diff": 0.006870000000000154, + "q50_diff": 0.012210000000000054, + "q75_diff": 0.02364999999999995, + "q95_diff": 0.04959000000000002 + }, + "P1_PIT02": { + "q05_diff": 0.03051999999999999, + "q25_diff": 0.10146999999999998, + "q50_diff": 0.10223999999999997, + "q75_diff": 0.0015300000000000313, + "q95_diff": 0.0022900000000000142 + }, + "P1_TIT01": { + "q05_diff": 0.06425000000000125, + "q25_diff": 0.07629999999999626, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.22887999999999664, + "q95_diff": 0.07629000000000019 + }, + "P1_TIT02": { + "q05_diff": 0.0, + "q25_diff": 0.01525999999999783, + "q50_diff": 0.13732999999999862, + "q75_diff": 0.25939000000000334, + "q95_diff": 0.4425099999999986 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.00013999999999825263, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.00010999999999938836 + }, + "P2_CO_rpm": { + "q05_diff": 11.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 2.0, + "q95_diff": 0.2656199999983073 + }, + "P2_HILout": { + "q05_diff": 0.3295900000000529, + "q25_diff": 1.6540499999999838, + "q50_diff": 0.31737999999995736, + "q75_diff": 1.7334000000000742, + "q95_diff": 2.2399900000000343 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 3.0, + "q25_diff": 2.0, + "q50_diff": 4.0, + "q75_diff": 6.0, + "q95_diff": 7.0 + }, + "P2_SIT02": { + "q05_diff": 2.6633900000000494, + "q25_diff": 3.5723900000000413, + "q50_diff": 5.610480000000052, + "q75_diff": 6.367129999999975, + "q95_diff": 6.833739999999921 + }, + "P2_VT01": { + "q05_diff": 0.00835000000000008, + "q25_diff": 0.015249999999999986, + "q50_diff": 0.014089999999999492, + "q75_diff": 0.013489999999999114, + "q95_diff": 0.018819999999999837 + }, + "P2_VXT02": { + "q05_diff": 0.13024000000000058, + "q25_diff": 0.29942, + "q50_diff": 0.28566, + "q75_diff": 0.26685000000000025, + "q95_diff": 0.45981000000000005 + }, + "P2_VXT03": { + "q05_diff": 0.10094999999999987, + "q25_diff": 0.26912999999999987, + "q50_diff": 0.29733999999999994, + "q75_diff": 0.32732000000000006, + "q95_diff": 0.61116 + }, + "P2_VYT02": { + "q05_diff": 0.09450000000000003, + "q25_diff": 0.26248000000000005, + "q50_diff": 0.22676000000000007, + "q75_diff": 0.2350399999999997, + "q95_diff": 0.38853000000000026 + }, + "P2_VYT03": { + "q05_diff": 0.05539999999999967, + "q25_diff": 0.18588000000000005, + "q50_diff": 0.17625000000000046, + "q75_diff": 0.18273000000000028, + "q95_diff": 0.35335999999999945 + }, + "P3_FIT01": { + "q05_diff": 3.0, + "q25_diff": 0.0, + "q50_diff": 70.0, + "q75_diff": 98.0, + "q95_diff": 114.0 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 103.36768000000006, + "q75_diff": 127.4433599999993, + "q95_diff": 11.193359999999302 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 256.0, + "q50_diff": 3824.0, + "q75_diff": 161.67871000000014, + "q95_diff": 31.040039999999863 + }, + "P3_LIT01": { + "q05_diff": 1134.7026400000004, + "q25_diff": 27.31151999999929, + "q50_diff": 650.7148400000005, + "q75_diff": 1140.0781199999983, + "q95_diff": 1.0507800000013958 + }, + "P3_PIT01": { + "q05_diff": 4.0, + "q25_diff": 5.0, + "q50_diff": 0.0, + "q75_diff": 928.0, + "q95_diff": 62.24805000000015 + }, + "P4_HT_FD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 7.000000000000002e-05, + "q75_diff": 7e-05, + "q95_diff": 0.0 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 27.14122, + "q75_diff": 0.13020999999999106, + "q95_diff": 0.21701000000000192 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 0.9223299999999999, + "q50_diff": 8.89752, + "q75_diff": 0.054199999999994475, + "q95_diff": 0.6510700000000043 + }, + "P4_LD": { + "q05_diff": 0.43831999999997606, + "q25_diff": 3.1083299999999667, + "q50_diff": 3.4359999999999786, + "q75_diff": 0.03201000000001386, + "q95_diff": 4.201050000000009 + }, + "P4_ST_FD": { + "q05_diff": 3.999999999999967e-05, + "q25_diff": 0.00013999999999999993, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 8.000000000000021e-05 + }, + "P4_ST_GOV": { + "q05_diff": 116.28319999999985, + "q25_diff": 38.044920000000275, + "q50_diff": 31.888670000000275, + "q75_diff": 110.5820300000014, + "q95_diff": 413.2324200000003 + }, + "P4_ST_LD": { + "q05_diff": 0.7356900000000337, + "q25_diff": 0.4004799999999591, + "q50_diff": 0.47867000000002236, + "q75_diff": 2.0575600000000236, + "q95_diff": 7.868069999999989 + }, + "P4_ST_PO": { + "q05_diff": 2.1548199999999724, + "q25_diff": 0.7244600000000219, + "q50_diff": 0.6116099999999847, + "q75_diff": 2.1616500000000087, + "q95_diff": 7.556669999999997 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 20.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 6.0, + "q50_diff": 3.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0624831607922858, + "P1_B2016": 0.06955732600083286, + "P1_B3004": 0.9420514424699565, + "P1_B3005": 0.9755864990985265, + "P1_B4002": 0.9939428775878422, + "P1_B4005": 1.0901635813805555, + "P1_B400B": 0.5854431525410038, + "P1_B4022": 1.0728287547128275, + "P1_FCV02Z": 1.0856923346443097, + "P1_FCV03D": 1.0287616800381814, + "P1_FCV03Z": 1.01920928123416, + "P1_FT01": 0.47270955964658845, + "P1_FT01Z": 0.36815633159095196, + "P1_FT02": 0.7916960456221444, + "P1_FT02Z": 1.0572315062387077, + "P1_FT03": 0.5895476319903267, + "P1_FT03Z": 0.9861265787637634, + "P1_LCV01D": 0.9922897795117445, + "P1_LCV01Z": 0.9035160861736835, + "P1_LIT01": 0.11337733428660046, + "P1_PCV01D": 0.9847169107083952, + "P1_PCV01Z": 1.1483081333334428, + "P1_PCV02Z": 0.5764046458797007, + "P1_PIT01": 0.1875869792291166, + "P1_PIT02": 0.19751880548222533, + "P1_TIT01": 0.04303664560561393, + "P1_TIT02": 0.08718233760664018, + "P2_24Vdc": 0.039965792976989475, + "P2_CO_rpm": 0.42742755199484195, + "P2_HILout": 0.24706059027478758, + "P2_MSD": 0.0, + "P2_SIT01": 0.43609660500873776, + "P2_SIT02": 0.5838584105971032, + "P2_VT01": 0.1492203579005219, + "P2_VXT02": 0.16161782957088977, + "P2_VXT03": 0.17996575274100857, + "P2_VYT02": 0.1464148112045175, + "P2_VYT03": 0.16868545803734847, + "P3_FIT01": 0.18381695503960072, + "P3_LCP01D": 0.20447780712073593, + "P3_LCV01D": 0.27862298184654877, + "P3_LIT01": 0.06847572452877615, + "P3_PIT01": 0.8461534722886025, + "P4_HT_FD": 0.03635399688775193, + "P4_HT_LD": 1.052071148480924, + "P4_HT_PO": 1.0918064404320895, + "P4_LD": 0.06943187697091346, + "P4_ST_FD": 0.28469546148549274, + "P4_ST_GOV": 0.0913873968521397, + "P4_ST_LD": 0.13676128749029393, + "P4_ST_PO": 0.08652073760594847, + "P4_ST_PT01": 0.6715692188345836, + "P4_ST_TT01": 0.1081291586433758 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.3825871639778149, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_temporal__seed1337/comprehensive_eval.json b/example/results/ablations/runs/no_temporal__seed1337/comprehensive_eval.json new file mode 100644 index 0000000..dfcfa49 --- /dev/null +++ b/example/results/ablations/runs/no_temporal__seed1337/comprehensive_eval.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_temporal__seed1337/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.40846108490566035, + "avg_jsd": 0.031160008477122598, + "avg_lag1_diff": 0.6642869349466621 + }, + "two_sample": { + "continuous_mmd_rbf": 0.5632306337356567, + "continuous_mmd_gamma": 1.6725705357622367e-05, + "histogram_mmd_rbf": 0.5632362365722656, + "histogram_mmd_gamma": 1.6724848669518688e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 162.63006591796875, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 2.947458458513477, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.30579161643981934, + "corr_frobenius": 22.44748878479004, + "lag1_corr_mean_abs_diff": 0.1933017522096634, + "lag1_corr_frobenius": 16.11225128173828, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.36698463559150696, + "corr_frobenius": 13.603052139282227, + "lag1_corr_mean_abs_diff": 0.35054484009742737, + "lag1_corr_frobenius": 12.259296417236328 + }, + "P2": { + "corr_mean_abs_diff": 0.23805589973926544, + "corr_frobenius": 3.5903096199035645, + "lag1_corr_mean_abs_diff": 0.17964617908000946, + "lag1_corr_frobenius": 3.237020492553711 + }, + "P3": { + "corr_mean_abs_diff": 0.5839857459068298, + "corr_frobenius": 4.082146644592285, + "lag1_corr_mean_abs_diff": 0.5411866903305054, + "lag1_corr_frobenius": 3.2619681358337402 + }, + "P4": { + "corr_mean_abs_diff": 0.3702239990234375, + "corr_frobenius": 5.349006175994873, + "lag1_corr_mean_abs_diff": 0.3339892327785492, + "lag1_corr_frobenius": 4.647590160369873 + } + } + }, + "frequency": { + "avg_psd_l1": 0.02569458819925785, + "avg_psd_cosine": 0.5474435314901669, + "avg_low_high_ratio_abs_diff": 9433977.554804089 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 15.0, + "mean_dwell": 24.0, + "median_dwell": 2.5, + "mean_step": 68.42660395304362, + "median_step": 1.9043693542480469 + }, + "P1_B2004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B3004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.01115664489586733, + "median_step": 0.004810333251953125 + }, + "P1_B3005": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22998.14285714286, + "mean_abs_diff_mean_dwell": 14201.625685203559, + "mean_abs_diff_median_dwell": 14207.642857142857, + "mean_abs_diff_mean_step": 33.693117622162895, + "mean_abs_diff_median_step": 0.831573486328125 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6041666666666666, + "change_rate": 0.741514360313316, + "step_median": 57.8821907043457 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.01520792643229163, + "mean_abs_diff_change_rate": 0.2806942429211713, + "mean_abs_diff_step_median": 57.85017040371895 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.0026041666666666665, + "top1_mass": 1.0, + "top3_mass": 1.0, + "median_dwell": 384.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.9869791666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.018229166666666668, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9192708333333334, + "top1_mass": 0.08333333333333333, + "top3_mass": 0.08854166666666667, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.18229166666666666, + "top1_mass": 0.15625, + "top3_mass": 0.3151041666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 1.0, + "top1_mass": 0.0026041666666666665, + "top3_mass": 0.0078125, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4557291666666667, + "top1_mass": 0.028645833333333332, + "top3_mass": 0.08333333333333333, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.9635416666666666, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.015625, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3125, + "top1_mass": 0.052083333333333336, + "top3_mass": 0.13020833333333334, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.571038564046224, + "mean_abs_diff_top1_mass": 0.08367792765299478, + "mean_abs_diff_top3_mass": 0.029201507568359382, + "mean_abs_diff_median_dwell": 47.75 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.1702899932861328, + "q50": 0.22292999923229218, + "q95": 0.22292999923229218, + "tail_ratio": 0.0 + }, + "P2_SIT02": { + "q05": 774.8919067382812, + "q50": 784.5540771484375, + "q95": 791.7107543945312, + "tail_ratio": 0.7406904393417769 + }, + "P1_FT03": { + "q05": 207.55763244628906, + "q50": 298.5381774902344, + "q95": 316.24639892578125, + "tail_ratio": 0.1946374516331318 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.1138192415237427, + "mean_abs_diff_q50": 2.37839903930823, + "mean_abs_diff_q95": 3.5791137466828027, + "mean_abs_diff_tail_ratio": 7.230527914021721 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 382.0, + "mean_dwell": 1.0026109660574412, + "median_dwell": 1.0, + "mean_step": 32.61420728274041, + "median_step": 18.502532958984375 + }, + "P1_FT02Z": { + "num_changes": 382.0, + "mean_dwell": 1.0026109660574412, + "median_dwell": 1.0, + "mean_step": 954.7115657042458, + "median_step": 846.0838470458984 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89280.5, + "mean_abs_diff_mean_dwell": 0.09831953427481777, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 486.0117859179777, + "mean_abs_diff_median_step": 429.02628326416016 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 32.7232666015625, + "std": 32.438350677490234, + "lag1": 0.11117122322320938 + }, + "P2_24Vdc": { + "mean": 28.028526306152344, + "std": 0.003789610927924514, + "lag1": -0.04372500255703926 + }, + "P2_HILout": { + "mean": 711.4883422851562, + "std": 9.425372123718262, + "lag1": 0.027140988036990166 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.7069098154703776, + "mean_abs_diff_std": 0.768325176400443, + "mean_abs_diff_lag1": 0.4065745851645867 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9770503044128418, + "mae": 0.7768316864967346 + }, + "real_plus_synthetic": { + "rmse": 0.5556530952453613, + "mae": 0.35953789949417114 + }, + "rmse_ratio_synth_to_real": 1.7498857228136122 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8273423005565863, + "auprc": 0.591814077952872, + "threshold": 0.028521830216050148, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6842105263157896, + "best_f1_threshold": 1.410032868385315 + }, + "real_plus_synthetic": { + "auroc": 0.8801716141001855, + "auprc": 0.6451048675574833, + "threshold": 0.3379894196987152, + "f1": 0.2909090909090909, + "precision": 0.18181818181818182, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.548900306224823 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_temporal__seed1337/comprehensive_eval_post.json b/example/results/ablations/runs/no_temporal__seed1337/comprehensive_eval_post.json new file mode 100644 index 0000000..7840e19 --- /dev/null +++ b/example/results/ablations/runs/no_temporal__seed1337/comprehensive_eval_post.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_temporal__seed1337/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.08156180381774902, + "continuous_mmd_gamma": 4.259624442557009e-05, + "histogram_mmd_rbf": 0.08163881301879883, + "histogram_mmd_gamma": 4.258310442142369e-05, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 92.75374603271484, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 1.6810410286670738, + "one_nn_two_sample_accuracy": 0.9959349593495935 + }, + "coupling": { + "corr_mean_abs_diff": 0.23099234700202942, + "corr_frobenius": 18.527013778686523, + "lag1_corr_mean_abs_diff": 0.18993210792541504, + "lag1_corr_frobenius": 16.22922134399414, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.3376525640487671, + "corr_frobenius": 11.97104263305664, + "lag1_corr_mean_abs_diff": 0.361699640750885, + "lag1_corr_frobenius": 12.659133911132812 + }, + "P2": { + "corr_mean_abs_diff": 0.16560013592243195, + "corr_frobenius": 2.721569061279297, + "lag1_corr_mean_abs_diff": 0.17755813896656036, + "lag1_corr_frobenius": 3.2211034297943115 + }, + "P3": { + "corr_mean_abs_diff": 0.5839857459068298, + "corr_frobenius": 4.082146644592285, + "lag1_corr_mean_abs_diff": 0.5411866903305054, + "lag1_corr_frobenius": 3.2619681358337402 + }, + "P4": { + "corr_mean_abs_diff": 0.3677639067173004, + "corr_frobenius": 5.335185527801514, + "lag1_corr_mean_abs_diff": 0.347543329000473, + "lag1_corr_frobenius": 4.863978385925293 + } + } + }, + "frequency": { + "avg_psd_l1": 0.026040498167276382, + "avg_psd_cosine": 0.5671232351154651, + "avg_low_high_ratio_abs_diff": 19.80377419982271 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 308.0, + "mean_dwell": 1.2427184466019416, + "median_dwell": 1.0, + "mean_step": 0.8413416007896523, + "median_step": 0.7274894714355469 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 347.0, + "mean_dwell": 1.103448275862069, + "median_dwell": 1.0, + "mean_step": 41.3141472878254, + "median_step": 54.116028785705566 + }, + "P1_B2004": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 0.014068839402653235, + "median_step": 0.0017450004816055298 + }, + "P1_B3004": { + "num_changes": 370.0, + "mean_dwell": 1.0350404312668464, + "median_dwell": 1.0, + "mean_step": 14.77256890374261, + "median_step": 10.804779052734375 + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.8157603410765646, + "median_step": 0.5944099426269531 + }, + "P1_B3005": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 64.28072175926926, + "median_step": 41.228179931640625 + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22997.85714285714, + "mean_abs_diff_mean_dwell": 14417.281142645239, + "mean_abs_diff_median_dwell": 13988.57142857143, + "mean_abs_diff_mean_step": 19.03385039702909, + "mean_abs_diff_median_step": 16.840154950817425 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6822916666666666, + "change_rate": 0.7702349869451697, + "step_median": 68.81373023986816 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.09333292643229163, + "mean_abs_diff_change_rate": 0.309414869553025, + "mean_abs_diff_step_median": 68.78170993924141 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.015625, + "top1_mass": 0.40625, + "top3_mass": 0.7760416666666666, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8776041666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.059895833333333336, + "top3_mass": 0.0703125, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.18229166666666666, + "top3_mass": 0.3697916666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.8463541666666666, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4817708333333333, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.7864583333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3515625, + "top1_mass": 0.036458333333333336, + "top3_mass": 0.09635416666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5303484598795573, + "mean_abs_diff_top1_mass": 0.03112411499023438, + "mean_abs_diff_top3_mass": 0.05629857381184896, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.1702899932861328, + "q50": 0.22292999923229218, + "q95": 0.22292999923229218, + "tail_ratio": 0.0 + }, + "P2_SIT02": { + "q05": 774.8919067382812, + "q50": 784.5540771484375, + "q95": 791.7107543945312, + "tail_ratio": 0.7406904393417769 + }, + "P1_FT03": { + "q05": 207.55763244628906, + "q50": 298.5381774902344, + "q95": 316.24639892578125, + "tail_ratio": 0.1946374516331318 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.1138192415237427, + "mean_abs_diff_q50": 2.37839903930823, + "mean_abs_diff_q95": 3.5791137466828027, + "mean_abs_diff_tail_ratio": 7.230527914021721 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 64.74391126130756, + "median_step": 62.410614013671875 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 1312.3134692358592, + "median_step": 1284.6439208984375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.09039718638964156, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 680.877589673068, + "mean_abs_diff_median_step": 670.2603607177734 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 35.254512786865234, + "std": 31.650897979736328, + "lag1": -0.0927804708480835 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.003297113813459873, + "lag1": 0.06643600016832352 + }, + "P2_HILout": { + "mean": 711.4987182617188, + "std": 7.856828212738037, + "lag1": -0.013758387416601181 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.4565467834472656, + "mean_abs_diff_std": 0.2373291968833655, + "mean_abs_diff_lag1": 0.4579590285817782 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.951804518699646, + "mae": 0.7649260759353638 + }, + "real_plus_synthetic": { + "rmse": 0.5429408550262451, + "mae": 0.34927552938461304 + }, + "rmse_ratio_synth_to_real": 1.7046708144499312 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8296150278293135, + "auprc": 0.5927880737400296, + "threshold": 0.007768048904836178, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6666666666666667, + "best_f1_threshold": 1.4181115627288818 + }, + "real_plus_synthetic": { + "auroc": 0.8809369202226345, + "auprc": 0.6459045059930429, + "threshold": 0.3188244104385376, + "f1": 0.2633744855967078, + "precision": 0.16080402010050251, + "recall": 0.7272727272727273, + "best_f1": 0.7532467532467532, + "best_f1_threshold": 0.4485895335674286 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_temporal__seed1337/eval.json b/example/results/ablations/runs/no_temporal__seed1337/eval.json new file mode 100644 index 0000000..fa02ab0 --- /dev/null +++ b/example/results/ablations/runs/no_temporal__seed1337/eval.json @@ -0,0 +1,974 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.11384, + "std": 0.0 + }, + "P1_B2016": { + "mean": 1.3915558854166672, + "std": 0.1394804798540304 + }, + "P1_B3004": { + "mean": 396.00211, + "std": 0.0 + }, + "P1_B3005": { + "mean": 1015.43976, + "std": 0.0 + }, + "P1_B4002": { + "mean": 33.54375, + "std": 0.0 + }, + "P1_B4005": { + "mean": 60.49572380208331, + "std": 42.2626602592901 + }, + "P1_B400B": { + "mean": 1773.2482367187495, + "std": 1201.4015313275845 + }, + "P1_B4022": { + "mean": 36.881583541666664, + "std": 0.14105102171131376 + }, + "P1_FCV02Z": { + "mean": 24.28350981770832, + "std": 39.81520081198682 + }, + "P1_FCV03D": { + "mean": 58.571172421875026, + "std": 6.640580036097716 + }, + "P1_FCV03Z": { + "mean": 59.261489192708346, + "std": 6.748260056909216 + }, + "P1_FT01": { + "mean": 194.94561549479187, + "std": 31.958255867612767 + }, + "P1_FT01Z": { + "mean": 878.5776119270837, + "std": 88.34502517192797 + }, + "P1_FT02": { + "mean": 1132.4921916927083, + "std": 890.6661188706163 + }, + "P1_FT02Z": { + "mean": 1132.4921916927083, + "std": 890.6661188706163 + }, + "P1_FT03": { + "mean": 282.48839567708313, + "std": 33.74219135359632 + }, + "P1_FT03Z": { + "mean": 282.48839567708313, + "std": 33.74219135359632 + }, + "P1_LCV01D": { + "mean": 12.166691979166664, + "std": 3.044583652481217 + }, + "P1_LCV01Z": { + "mean": 11.953970416666662, + "std": 3.1112112025057415 + }, + "P1_LIT01": { + "mean": 404.113070078125, + "std": 15.916941648124695 + }, + "P1_PCV01D": { + "mean": 67.44947348958331, + "std": 23.853487324941693 + }, + "P1_PCV01Z": { + "mean": 68.38945937500003, + "std": 23.945563502317672 + }, + "P1_PCV02Z": { + "mean": 11.76605, + "std": 0.0 + }, + "P1_PIT01": { + "mean": 1.3766178124999982, + "std": 0.08110125251296071 + }, + "P1_PIT02": { + "mean": 0.2946465364583334, + "std": 0.36556687150544787 + }, + "P1_TIT01": { + "mean": 35.62746005208336, + "std": 0.567873449151457 + }, + "P1_TIT02": { + "mean": 36.50254283854163, + "std": 1.2327469725698657 + }, + "P2_24Vdc": { + "mean": 28.028526171875, + "std": 0.003789607636453064 + }, + "P2_CO_rpm": { + "mean": 54102.3476257292, + "std": 24.202381763958453 + }, + "P2_HILout": { + "mean": 711.4883096354174, + "std": 9.425371538528266 + }, + "P2_MSD": { + "mean": 758.4425, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.3845312500002, + "std": 6.124744810717389 + }, + "P2_SIT02": { + "mean": 783.9236397916661, + "std": 5.839287889306126 + }, + "P2_VT01": { + "mean": 11.923775807291669, + "std": 0.06899660403626517 + }, + "P2_VXT02": { + "mean": -3.2993178125, + "std": 0.4772571828037386 + }, + "P2_VXT03": { + "mean": -1.3056295312500004, + "std": 0.35753695718235284 + }, + "P2_VYT02": { + "mean": 3.9807949739583326, + "std": 0.5057799909448849 + }, + "P2_VYT03": { + "mean": 6.233346276041665, + "std": 0.38919692566379294 + }, + "P3_FIT01": { + "mean": 1155.0978390364578, + "std": 1787.3394963520552 + }, + "P3_LCP01D": { + "mean": 4575.033106822919, + "std": 5118.60925242513 + }, + "P3_LCV01D": { + "mean": 6456.875321927081, + "std": 6929.023824930554 + }, + "P3_LIT01": { + "mean": 13352.48445802083, + "std": 4371.061885508871 + }, + "P3_PIT01": { + "mean": 652.8043708333335, + "std": 1157.5726463848948 + }, + "P4_HT_FD": { + "mean": -0.00120078125, + "std": 0.005632729922998602 + }, + "P4_HT_LD": { + "mean": -115.69799739583333, + "std": 35.27327640165136 + }, + "P4_HT_PO": { + "mean": 32.723265390625, + "std": 32.43835192744646 + }, + "P4_LD": { + "mean": 366.1244961458333, + "std": 61.02708375047362 + }, + "P4_ST_FD": { + "mean": -6.260416666666658e-05, + "std": 0.004046906369568698 + }, + "P4_ST_GOV": { + "mean": 17918.57285067709, + "std": 1897.0060826934864 + }, + "P4_ST_LD": { + "mean": 332.0924436979168, + "std": 38.59368587774719 + }, + "P4_ST_PO": { + "mean": 332.3720863020833, + "std": 35.274321435734834 + }, + "P4_ST_PT01": { + "mean": 10044.738232942711, + "std": 28.381219888425726 + }, + "P4_ST_TT01": { + "mean": 27608.46886190105, + "std": 24.259452212920237 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.020272347909024324, + "std_abs_err": 0.017998437082092405 + }, + "P1_B2016": { + "mean_abs_err": 0.0002895607519002308, + "std_abs_err": 0.0037069173774673114 + }, + "P1_B3004": { + "mean_abs_err": 7.383953824526145, + "std_abs_err": 15.41814958804181 + }, + "P1_B3005": { + "mean_abs_err": 45.609026528629784, + "std_abs_err": 61.18331651100028 + }, + "P1_B4002": { + "mean_abs_err": 1.2268236801201198, + "std_abs_err": 0.5786734991990778 + }, + "P1_B4005": { + "mean_abs_err": 0.029383214114609757, + "std_abs_err": 0.08136281158041214 + }, + "P1_B400B": { + "mean_abs_err": 5.227633459218623, + "std_abs_err": 8.049258408511832 + }, + "P1_B4022": { + "mean_abs_err": 0.8204088942641405, + "std_abs_err": 0.5303998331235046 + }, + "P1_FCV02Z": { + "mean_abs_err": 0.6674250827639554, + "std_abs_err": 0.6232846272713601 + }, + "P1_FCV03D": { + "mean_abs_err": 0.0033089710281970497, + "std_abs_err": 0.04108431738896101 + }, + "P1_FCV03Z": { + "mean_abs_err": 0.03754998748149063, + "std_abs_err": 0.03345609257761417 + }, + "P1_FT01": { + "mean_abs_err": 0.22196113851160248, + "std_abs_err": 4.6102740289684405 + }, + "P1_FT01Z": { + "mean_abs_err": 0.4720695770688508, + "std_abs_err": 12.579322208609724 + }, + "P1_FT02": { + "mean_abs_err": 0.3259615419178772, + "std_abs_err": 1.4699550192036668 + }, + "P1_FT02Z": { + "mean_abs_err": 645.9862549159404, + "std_abs_err": 302.6792369005333 + }, + "P1_FT03": { + "mean_abs_err": 0.1505745991457843, + "std_abs_err": 0.26797824230081346 + }, + "P1_FT03Z": { + "mean_abs_err": 778.5536426714384, + "std_abs_err": 27.38329751002101 + }, + "P1_LCV01D": { + "mean_abs_err": 0.01235931739152818, + "std_abs_err": 0.12996869945109557 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.012103612671255704, + "std_abs_err": 0.14774170516042995 + }, + "P1_LIT01": { + "mean_abs_err": 0.0018020848812625445, + "std_abs_err": 0.2850435997507237 + }, + "P1_PCV01D": { + "mean_abs_err": 0.01610024610806704, + "std_abs_err": 0.08372062403611835 + }, + "P1_PCV01Z": { + "mean_abs_err": 0.018692421902557044, + "std_abs_err": 0.08096369165402706 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.07670294660492694, + "std_abs_err": 0.09684446546464737 + }, + "P1_PIT01": { + "mean_abs_err": 0.0008634082533938159, + "std_abs_err": 0.019668693143078612 + }, + "P1_PIT02": { + "mean_abs_err": 0.19251055388318333, + "std_abs_err": 0.13314476654943197 + }, + "P1_TIT01": { + "mean_abs_err": 0.05374819034985734, + "std_abs_err": 0.015407178034491986 + }, + "P1_TIT02": { + "mean_abs_err": 0.022435839794901824, + "std_abs_err": 0.010988246545602598 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00018353075266830388, + "std_abs_err": 0.0004471415220461488 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.36257215347723104, + "std_abs_err": 1.7466499891700877 + }, + "P2_HILout": { + "mean_abs_err": 0.025500365321249774, + "std_abs_err": 0.970645661272103 + }, + "P2_MSD": { + "mean_abs_err": 4.750739999999951, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.5193861634427321, + "std_abs_err": 0.9258294502731639 + }, + "P2_SIT02": { + "mean_abs_err": 0.019901909119880656, + "std_abs_err": 0.6487081324931596 + }, + "P2_VT01": { + "mean_abs_err": 3.983871846457987e-05, + "std_abs_err": 0.00048292196426311107 + }, + "P2_VXT02": { + "mean_abs_err": 0.0008606598899962847, + "std_abs_err": 0.006224256532518291 + }, + "P2_VXT03": { + "mean_abs_err": 0.001263755414677492, + "std_abs_err": 0.007909578018699182 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006738760346043371, + "std_abs_err": 0.005869537952097048 + }, + "P2_VYT03": { + "mean_abs_err": 0.0010588367562194279, + "std_abs_err": 0.009393963692701346 + }, + "P3_FIT01": { + "mean_abs_err": 4.197662930274646, + "std_abs_err": 7.61479856178039 + }, + "P3_LCP01D": { + "mean_abs_err": 9.694041417655171, + "std_abs_err": 11.390745290748782 + }, + "P3_LCV01D": { + "mean_abs_err": 677.0013950541606, + "std_abs_err": 89.09592911473283 + }, + "P3_LIT01": { + "mean_abs_err": 2.360034673156406, + "std_abs_err": 16.887248580934283 + }, + "P3_PIT01": { + "mean_abs_err": 3.4464678100057426, + "std_abs_err": 6.650715677699509 + }, + "P4_HT_FD": { + "mean_abs_err": 0.0010975933046482602, + "std_abs_err": 0.003600711608594502 + }, + "P4_HT_LD": { + "mean_abs_err": 149.69291447551893, + "std_abs_err": 1.8494828916530537 + }, + "P4_HT_PO": { + "mean_abs_err": 1.260542404131904, + "std_abs_err": 0.9974807101249858 + }, + "P4_LD": { + "mean_abs_err": 0.004257648361033262, + "std_abs_err": 0.4112773810306152 + }, + "P4_ST_FD": { + "mean_abs_err": 3.349093678351776e-05, + "std_abs_err": 0.002404730111578669 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.611589429565356, + "std_abs_err": 44.86286670571303 + }, + "P4_ST_LD": { + "mean_abs_err": 0.08954652809910613, + "std_abs_err": 0.7405165861296439 + }, + "P4_ST_PO": { + "mean_abs_err": 0.10311497842451445, + "std_abs_err": 0.8233779333265332 + }, + "P4_ST_PT01": { + "mean_abs_err": 5.836923876371657, + "std_abs_err": 10.596449863106596 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.5638297784826136, + "std_abs_err": 0.34213040835190967 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 1.0, + "P1_B2016": 0.043391666666666606, + "P1_B3004": 0.8108, + "P1_B3005": 0.8106, + "P1_B4002": 0.81055, + "P1_B4005": 0.38133333333333336, + "P1_B400B": 0.46184583333333334, + "P1_B4022": 0.5730333333333334, + "P1_FCV02Z": 0.7381416666666666, + "P1_FCV03D": 0.4878416666666667, + "P1_FCV03Z": 0.4924416666666667, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4750083333333333, + "P1_FT02": 0.4545666666666666, + "P1_FT02Z": 0.8361000000000001, + "P1_FT03": 0.5241, + "P1_FT03Z": 1.0, + "P1_LCV01D": 0.44738750000000005, + "P1_LCV01Z": 0.4546666666666667, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5571583333333334, + "P1_PCV01Z": 0.5587541666666667, + "P1_PCV02Z": 1.0, + "P1_PIT01": 0.17675833333333335, + "P1_PIT02": 0.8118208333333334, + "P1_TIT01": 0.16881250000000003, + "P1_TIT02": 0.10090416666666668, + "P2_24Vdc": 0.12129166666666658, + "P2_CO_rpm": 0.0927541666666667, + "P2_HILout": 0.08199583333333332, + "P2_MSD": 1.0, + "P2_SIT01": 0.48742916666666664, + "P2_SIT02": 0.5013166666666666, + "P2_VT01": 0.12336250000000004, + "P2_VXT02": 0.28222916666666664, + "P2_VXT03": 0.43657500000000005, + "P2_VYT02": 0.25022083333333334, + "P2_VYT03": 0.26955833333333334, + "P3_FIT01": 0.0821, + "P3_LCP01D": 0.04134166666666661, + "P3_LCV01D": 0.235225, + "P3_LIT01": 0.09348749999999995, + "P3_PIT01": 0.1158541666666667, + "P4_HT_FD": 0.5712458333333333, + "P4_HT_LD": 0.9375, + "P4_HT_PO": 0.3441416666666667, + "P4_LD": 0.031329166666666686, + "P4_ST_FD": 0.024483333333333412, + "P4_ST_GOV": 0.035274999999999945, + "P4_ST_LD": 0.03237500000000004, + "P4_ST_PO": 0.03612083333333338, + "P4_ST_PT01": 0.20012916666666664, + "P4_ST_TT01": 0.07880833333333337 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.08333, + "q25_diff": 0.012630000000000002, + "q50_diff": 0.012630000000000002, + "q75_diff": 0.012630000000000002, + "q95_diff": 0.012630000000000002 + }, + "P1_B2016": { + "q05_diff": 0.01158999999999999, + "q25_diff": 0.010220000000000118, + "q50_diff": 0.005319999999999991, + "q75_diff": 0.014129999999999976, + "q95_diff": 0.02967999999999993 + }, + "P1_B3004": { + "q05_diff": 2.8398500000000126, + "q25_diff": 1.6357399999999984, + "q50_diff": 1.6357399999999984, + "q75_diff": 1.6357399999999984, + "q95_diff": 1.6357399999999984 + }, + "P1_B3005": { + "q05_diff": 13.44177000000002, + "q25_diff": 13.44177000000002, + "q50_diff": 13.44177000000002, + "q75_diff": 13.44177000000002, + "q95_diff": 99.67999000000009 + }, + "P1_B4002": { + "q05_diff": 1.5437500000000028, + "q25_diff": 0.11175000000000068, + "q50_diff": 0.11175000000000068, + "q75_diff": 0.11175000000000068, + "q95_diff": 0.11175000000000068 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 88.15009, + "q50_diff": 20.33117, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 2.9795300000000005, + "q25_diff": 2610.38788, + "q50_diff": 189.59229000000005, + "q75_diff": 9.545890000000327, + "q95_diff": 7.019040000000132 + }, + "P1_B4022": { + "q05_diff": 1.2294699999999992, + "q25_diff": 0.1574500000000043, + "q50_diff": 0.4875299999999996, + "q75_diff": 0.5946499999999943, + "q95_diff": 0.6461800000000011 + }, + "P1_FCV02Z": { + "q05_diff": 0.015249999999999986, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.06867000000000001, + "q75_diff": 41.04615, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 2.8851799999999983, + "q25_diff": 0.09517999999999915, + "q50_diff": 8.548209999999997, + "q75_diff": 11.145269999999996, + "q95_diff": 0.715919999999997 + }, + "P1_FCV03Z": { + "q05_diff": 3.2806399999999982, + "q25_diff": 0.01525999999999783, + "q50_diff": 8.735659999999996, + "q75_diff": 11.566150000000007, + "q95_diff": 0.41956000000000415 + }, + "P1_FT01": { + "q05_diff": 8.109370000000013, + "q25_diff": 8.601060000000018, + "q50_diff": 32.806329999999974, + "q75_diff": 37.95618999999999, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 38.15130999999997, + "q25_diff": 18.69965000000002, + "q50_diff": 74.79840000000002, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.23904, + "q50_diff": 258.24926000000005, + "q75_diff": 12.970209999999952, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 27.82217, + "q25_diff": 2761.17983, + "q50_diff": 1117.82361, + "q75_diff": 870.8322700000001, + "q95_diff": 865.2337600000001 + }, + "P1_FT03": { + "q05_diff": 37.54695000000001, + "q25_diff": 1.5381300000000238, + "q50_diff": 51.33557000000002, + "q75_diff": 59.82164, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 790.28081, + "q25_diff": 752.66058, + "q50_diff": 704.08377, + "q75_diff": 697.55072, + "q95_diff": 800.17028 + }, + "P1_LCV01D": { + "q05_diff": 0.10452999999999957, + "q25_diff": 1.2063699999999997, + "q50_diff": 4.634790000000001, + "q75_diff": 3.3112399999999997, + "q95_diff": 0.6190800000000003 + }, + "P1_LCV01Z": { + "q05_diff": 0.1144400000000001, + "q25_diff": 1.1901799999999998, + "q50_diff": 4.97481, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.7171600000000016 + }, + "P1_LIT01": { + "q05_diff": 14.395199999999988, + "q25_diff": 1.0270700000000375, + "q50_diff": 6.185159999999996, + "q75_diff": 8.229829999999993, + "q95_diff": 34.807680000000005 + }, + "P1_PCV01D": { + "q05_diff": 1.0672799999999967, + "q25_diff": 3.7126699999999957, + "q50_diff": 22.269549999999995, + "q75_diff": 44.83369, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.9702200000000047, + "q25_diff": 3.814699999999995, + "q50_diff": 22.768570000000004, + "q75_diff": 45.46356000000001, + "q95_diff": 45.146029999999996 + }, + "P1_PCV02Z": { + "q05_diff": 0.2441399999999998, + "q25_diff": 0.2441399999999998, + "q50_diff": 0.2517700000000005, + "q75_diff": 0.2517700000000005, + "q95_diff": 0.2517700000000005 + }, + "P1_PIT01": { + "q05_diff": 0.035090000000000066, + "q25_diff": 0.016790000000000083, + "q50_diff": 0.006099999999999994, + "q75_diff": 0.02364999999999995, + "q95_diff": 0.008410000000000029 + }, + "P1_PIT02": { + "q05_diff": 0.03433, + "q25_diff": 0.05263999999999999, + "q50_diff": 0.053409999999999985, + "q75_diff": 0.054170000000000024, + "q95_diff": 0.05493000000000001 + }, + "P1_TIT01": { + "q05_diff": 0.07092999999999705, + "q25_diff": 0.07629999999999626, + "q50_diff": 0.15259000000000356, + "q75_diff": 0.3509600000000006, + "q95_diff": 0.01525999999999783 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.062220000000003495, + "q50_diff": 0.10680999999999585, + "q75_diff": 0.2908199999999965, + "q95_diff": 0.4344200000000029 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.0008799999999986596, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.000920000000000698 + }, + "P2_CO_rpm": { + "q05_diff": 10.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 3.0, + "q95_diff": 0.33202999999775784 + }, + "P2_HILout": { + "q05_diff": 0.6367199999999684, + "q25_diff": 2.0198899999999185, + "q50_diff": 0.08379999999999654, + "q75_diff": 1.5365600000000086, + "q95_diff": 1.1483800000000883 + }, + "P2_MSD": { + "q05_diff": 4.750739999999951, + "q25_diff": 4.750739999999951, + "q50_diff": 4.750739999999951, + "q75_diff": 4.750739999999951, + "q95_diff": 4.750739999999951 + }, + "P2_SIT01": { + "q05_diff": 2.0, + "q25_diff": 3.0, + "q50_diff": 4.5, + "q75_diff": 6.0, + "q95_diff": 6.0 + }, + "P2_SIT02": { + "q05_diff": 2.6317200000000867, + "q25_diff": 3.570310000000063, + "q50_diff": 5.6193300000001045, + "q75_diff": 6.361210000000028, + "q95_diff": 6.843689999999924 + }, + "P2_VT01": { + "q05_diff": 0.008480000000000487, + "q25_diff": 0.015229999999998967, + "q50_diff": 0.014120000000000132, + "q75_diff": 0.013400000000000745, + "q95_diff": 0.01904000000000039 + }, + "P2_VXT02": { + "q05_diff": 0.12995, + "q25_diff": 0.29942, + "q50_diff": 0.2857599999999998, + "q75_diff": 0.26654, + "q95_diff": 0.45904999999999996 + }, + "P2_VXT03": { + "q05_diff": 0.10075999999999996, + "q25_diff": 0.2691999999999999, + "q50_diff": 0.2973199999999998, + "q75_diff": 0.32702999999999993, + "q95_diff": 0.61067 + }, + "P2_VYT02": { + "q05_diff": 0.09643999999999986, + "q25_diff": 0.26202000000000014, + "q50_diff": 0.22612000000000032, + "q75_diff": 0.2354699999999994, + "q95_diff": 0.3892100000000003 + }, + "P2_VYT03": { + "q05_diff": 0.055449999999999555, + "q25_diff": 0.1862699999999995, + "q50_diff": 0.17562000000000033, + "q75_diff": 0.18233999999999995, + "q95_diff": 0.35440000000000005 + }, + "P3_FIT01": { + "q05_diff": 1.0, + "q25_diff": 0.0, + "q50_diff": 48.63019, + "q75_diff": 81.7773400000001, + "q95_diff": 18.52930000000015 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 95.69885, + "q75_diff": 133.79297000000042, + "q95_diff": 240.0 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 320.0, + "q50_diff": 3189.4414100000004, + "q75_diff": 175.13378999999986, + "q95_diff": 23.926760000000286 + }, + "P3_LIT01": { + "q05_diff": 1135.7871100000002, + "q25_diff": 26.398440000000846, + "q50_diff": 650.5126999999993, + "q75_diff": 1148.9355499999983, + "q95_diff": 1.3242199999986042 + }, + "P3_PIT01": { + "q05_diff": 1.0, + "q25_diff": 0.0, + "q50_diff": 1.0, + "q75_diff": 79.31182999999999, + "q95_diff": 62.41211000000021 + }, + "P4_HT_FD": { + "q05_diff": 0.00174, + "q25_diff": 0.00043000000000000004, + "q50_diff": 0.0005700000000000001, + "q75_diff": 7e-05, + "q95_diff": 0.0005099999999999999 + }, + "P4_HT_LD": { + "q05_diff": 124.7862, + "q25_diff": 124.7862, + "q50_diff": 179.53157, + "q75_diff": 193.36982, + "q95_diff": 56.57957999999999 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 1.35638, + "q50_diff": 30.852110000000003, + "q75_diff": 0.7969400000000064, + "q95_diff": 0.10701000000000249 + }, + "P4_LD": { + "q05_diff": 0.5629900000000134, + "q25_diff": 3.0934100000000058, + "q50_diff": 3.561399999999992, + "q75_diff": 0.026119999999991705, + "q95_diff": 4.2164600000000405 + }, + "P4_ST_FD": { + "q05_diff": 2.9999999999999645e-05, + "q25_diff": 0.0, + "q50_diff": 7.999999999999999e-05, + "q75_diff": 7.000000000000008e-05, + "q95_diff": 0.00014999999999999996 + }, + "P4_ST_GOV": { + "q05_diff": 128.33691, + "q25_diff": 38.02538999999888, + "q50_diff": 32.49022999999943, + "q75_diff": 105.10155999999915, + "q95_diff": 414.03515999999945 + }, + "P4_ST_LD": { + "q05_diff": 0.7499099999999999, + "q25_diff": 0.48076999999994996, + "q50_diff": 0.38602000000003045, + "q75_diff": 2.0497100000000046, + "q95_diff": 7.9558399999999665 + }, + "P4_ST_PO": { + "q05_diff": 2.1662000000000035, + "q25_diff": 0.725529999999992, + "q50_diff": 0.6129799999999932, + "q75_diff": 2.143250000000023, + "q95_diff": 7.837030000000027 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 14.0, + "q50_diff": 2.0, + "q75_diff": 0.0, + "q95_diff": 30.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 6.0, + "q50_diff": 3.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0189476811306442e-05, + "P1_B2016": 0.2561068038657559, + "P1_B3004": 1.018740504876714e-05, + "P1_B3005": 0.9999898125942444, + "P1_B4002": 0.9999898146591784, + "P1_B4005": 0.8705270534118652, + "P1_B400B": 1.0636685342327254, + "P1_B4022": 0.03037469907643997, + "P1_FCV02Z": 1.0359760160093519, + "P1_FCV03D": 0.9348723916270425, + "P1_FCV03Z": 0.9302223036218717, + "P1_FT01": 1.0837129234395464, + "P1_FT01Z": 0.8647923079278955, + "P1_FT02": 1.0102725969371698, + "P1_FT02Z": 1.0103275755470813, + "P1_FT03": 0.8468517424320046, + "P1_FT03Z": 0.8467146853873977, + "P1_LCV01D": 0.4377534512779314, + "P1_LCV01Z": 0.44515898519261965, + "P1_LIT01": 0.963461971457682, + "P1_PCV01D": 1.084519658135186, + "P1_PCV01Z": 0.8997968894051388, + "P1_PCV02Z": 0.5669400491575541, + "P1_PIT01": 0.4203947521554928, + "P1_PIT02": 0.9828387650450174, + "P1_TIT01": 0.7798971257915577, + "P1_TIT02": 0.865590207042142, + "P2_24Vdc": 0.0701931045432132, + "P2_CO_rpm": 0.27366047721795406, + "P2_HILout": 0.20616114471773853, + "P2_MSD": 0.0, + "P2_SIT01": 0.6624061266276653, + "P2_SIT02": 0.625353002969849, + "P2_VT01": 0.44589844834894954, + "P2_VXT02": 0.024527633833306917, + "P2_VXT03": 0.26813324104938296, + "P2_VYT02": 0.5193038198965017, + "P2_VYT03": 0.39748846203270954, + "P3_FIT01": 0.8757503767058722, + "P3_LCP01D": 0.956442423364842, + "P3_LCV01D": 0.845280051194023, + "P3_LIT01": 0.9604496464042352, + "P3_PIT01": 0.9121126092062325, + "P4_HT_FD": 0.33671422249468896, + "P4_HT_LD": 0.15670838558697175, + "P4_HT_PO": 0.8878547296538263, + "P4_LD": 0.8941496210667775, + "P4_ST_FD": 0.29904099302570664, + "P4_ST_GOV": 0.8465337816178481, + "P4_ST_LD": 0.8603354377847401, + "P4_ST_PO": 0.7944502897383621, + "P4_ST_PT01": 0.9795661975361911, + "P4_ST_TT01": 0.8779218332437576 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.6325772640315827, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + }, + "avg_ks": 0.40846108490566035, + "avg_jsd": 0.031160008477122598, + "avg_lag1_diff": 0.6642869349466621 +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_temporal__seed1337/eval_post.json b/example/results/ablations/runs/no_temporal__seed1337/eval_post.json new file mode 100644 index 0000000..309f7cf --- /dev/null +++ b/example/results/ablations/runs/no_temporal__seed1337/eval_post.json @@ -0,0 +1,971 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09355486979166669, + "std": 0.01799194293084728 + }, + "P1_B2016": { + "mean": 1.3915558854166672, + "std": 0.1394804798540304 + }, + "P1_B3004": { + "mean": 403.2905014583335, + "std": 14.217563527951576 + }, + "P1_B3005": { + "mean": 1059.5253898958326, + "std": 58.42587703751758 + }, + "P1_B4002": { + "mean": 32.37490617187499, + "std": 0.6519533213854061 + }, + "P1_B4005": { + "mean": 64.99571291666655, + "std": 42.03467331899641 + }, + "P1_B400B": { + "mean": 1773.2482367187495, + "std": 1201.4015313275845 + }, + "P1_B4022": { + "mean": 36.08671476562498, + "std": 0.7147782383668128 + }, + "P1_FCV02Z": { + "mean": 21.954832630208315, + "std": 38.87335152465215 + }, + "P1_FCV03D": { + "mean": 58.14541601562505, + "std": 6.267267768996455 + }, + "P1_FCV03Z": { + "mean": 58.21398343750003, + "std": 6.423211358701469 + }, + "P1_FT01": { + "mean": 194.94561549479187, + "std": 31.958255867612767 + }, + "P1_FT01Z": { + "mean": 878.5776119270837, + "std": 88.34502517192797 + }, + "P1_FT02": { + "mean": 1132.4921916927083, + "std": 890.6661188706163 + }, + "P1_FT02Z": { + "mean": 1843.7261292187509, + "std": 1190.4241853713818 + }, + "P1_FT03": { + "mean": 282.48839567708313, + "std": 33.74219135359632 + }, + "P1_FT03Z": { + "mean": 1058.2564329947922, + "std": 59.50957255113764 + }, + "P1_LCV01D": { + "mean": 11.699055963541667, + "std": 3.0155193220433567 + }, + "P1_LCV01Z": { + "mean": 11.627510442708333, + "std": 3.0251072332459787 + }, + "P1_LIT01": { + "mean": 404.113070078125, + "std": 15.916941648124695 + }, + "P1_PCV01D": { + "mean": 65.64593611979173, + "std": 23.56629129628467 + }, + "P1_PCV01Z": { + "mean": 65.565628671875, + "std": 23.790956943468952 + }, + "P1_PCV02Z": { + "mean": 11.867279765625007, + "std": 0.10811550492611105 + }, + "P1_PIT01": { + "mean": 1.3766178124999982, + "std": 0.08110125251296071 + }, + "P1_PIT02": { + "mean": 0.2946465364583334, + "std": 0.36556687150544787 + }, + "P1_TIT01": { + "mean": 35.62746005208336, + "std": 0.567873449151457 + }, + "P1_TIT02": { + "mean": 36.50254283854163, + "std": 1.2327469725698657 + }, + "P2_24Vdc": { + "mean": 28.02860809895833, + "std": 0.0032970984602375326 + }, + "P2_CO_rpm": { + "mean": 54102.3476257292, + "std": 24.202381763958453 + }, + "P2_HILout": { + "mean": 711.4986896354167, + "std": 7.856828438379631 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.3845312500002, + "std": 6.124744810717389 + }, + "P2_SIT02": { + "mean": 783.9236397916661, + "std": 5.839287889306126 + }, + "P2_VT01": { + "mean": 11.923775807291669, + "std": 0.06899660403626517 + }, + "P2_VXT02": { + "mean": -3.2993178125, + "std": 0.4772571828037386 + }, + "P2_VXT03": { + "mean": -1.3056295312500004, + "std": 0.35753695718235284 + }, + "P2_VYT02": { + "mean": 3.9807949739583326, + "std": 0.5057799909448849 + }, + "P2_VYT03": { + "mean": 6.233346276041665, + "std": 0.38919692566379294 + }, + "P3_FIT01": { + "mean": 1155.0978390364578, + "std": 1787.3394963520552 + }, + "P3_LCP01D": { + "mean": 4575.033106822919, + "std": 5118.60925242513 + }, + "P3_LCV01D": { + "mean": 6456.875321927081, + "std": 6929.023824930554 + }, + "P3_LIT01": { + "mean": 13352.48445802083, + "std": 4371.061885508871 + }, + "P3_PIT01": { + "mean": 652.8043708333335, + "std": 1157.5726463848948 + }, + "P4_HT_FD": { + "mean": -0.00120078125, + "std": 0.005632729922998602 + }, + "P4_HT_LD": { + "mean": 34.231568880208286, + "std": 33.60361854009684 + }, + "P4_HT_PO": { + "mean": 35.25451283854166, + "std": 31.650898099038645 + }, + "P4_LD": { + "mean": 366.1244961458333, + "std": 61.02708375047362 + }, + "P4_ST_FD": { + "mean": -6.260416666666658e-05, + "std": 0.004046906369568698 + }, + "P4_ST_GOV": { + "mean": 17918.57285067709, + "std": 1897.0060826934864 + }, + "P4_ST_LD": { + "mean": 332.0924436979168, + "std": 38.59368587774719 + }, + "P4_ST_PO": { + "mean": 332.3720863020833, + "std": 35.274321435734834 + }, + "P4_ST_PT01": { + "mean": 10044.738232942711, + "std": 28.381219888425726 + }, + "P4_ST_TT01": { + "mean": 27608.46886190105, + "std": 24.259452212920237 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 1.2782299308983225e-05, + "std_abs_err": 6.494151245123314e-06 + }, + "P1_B2016": { + "mean_abs_err": 0.0002895607519002308, + "std_abs_err": 0.0037069173774673114 + }, + "P1_B3004": { + "mean_abs_err": 0.09556236619266656, + "std_abs_err": 1.2005860600902345 + }, + "P1_B3005": { + "mean_abs_err": 1.5233966327971302, + "std_abs_err": 2.757439473482698 + }, + "P1_B4002": { + "mean_abs_err": 0.05797985199510691, + "std_abs_err": 0.07327982218632823 + }, + "P1_B4005": { + "mean_abs_err": 4.470605900468627, + "std_abs_err": 0.14662412871327746 + }, + "P1_B400B": { + "mean_abs_err": 5.227633459218623, + "std_abs_err": 8.049258408511832 + }, + "P1_B4022": { + "mean_abs_err": 0.02554011822245883, + "std_abs_err": 0.04332738353199439 + }, + "P1_FCV02Z": { + "mean_abs_err": 2.996102270263961, + "std_abs_err": 1.5651339146060295 + }, + "P1_FCV03D": { + "mean_abs_err": 0.42244743522177686, + "std_abs_err": 0.3322279497122995 + }, + "P1_FCV03Z": { + "mean_abs_err": 1.0850557426898035, + "std_abs_err": 0.29159260563013323 + }, + "P1_FT01": { + "mean_abs_err": 0.22196113851160248, + "std_abs_err": 4.6102740289684405 + }, + "P1_FT01Z": { + "mean_abs_err": 0.4720695770688508, + "std_abs_err": 12.579322208609724 + }, + "P1_FT02": { + "mean_abs_err": 0.3259615419178772, + "std_abs_err": 1.4699550192036668 + }, + "P1_FT02Z": { + "mean_abs_err": 65.24768261010217, + "std_abs_err": 2.9211703997677887 + }, + "P1_FT03": { + "mean_abs_err": 0.1505745991457843, + "std_abs_err": 0.26797824230081346 + }, + "P1_FT03Z": { + "mean_abs_err": 2.785605353729352, + "std_abs_err": 1.6159163124796905 + }, + "P1_LCV01D": { + "mean_abs_err": 0.45527669823346884, + "std_abs_err": 0.1009043690132354 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.33856358662958463, + "std_abs_err": 0.061637735900667234 + }, + "P1_LIT01": { + "mean_abs_err": 0.0018020848812625445, + "std_abs_err": 0.2850435997507237 + }, + "P1_PCV01D": { + "mean_abs_err": 1.8196376158996515, + "std_abs_err": 0.20347540462090308 + }, + "P1_PCV01Z": { + "mean_abs_err": 2.842523125027583, + "std_abs_err": 0.07364286719469249 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.024526819020080026, + "std_abs_err": 0.011271039461463683 + }, + "P1_PIT01": { + "mean_abs_err": 0.0008634082533938159, + "std_abs_err": 0.019668693143078612 + }, + "P1_PIT02": { + "mean_abs_err": 0.19251055388318333, + "std_abs_err": 0.13314476654943197 + }, + "P1_TIT01": { + "mean_abs_err": 0.05374819034985734, + "std_abs_err": 0.015407178034491986 + }, + "P1_TIT02": { + "mean_abs_err": 0.022435839794901824, + "std_abs_err": 0.010988246545602598 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00010160366933931186, + "std_abs_err": 4.536765416938244e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.36257215347723104, + "std_abs_err": 1.7466499891700877 + }, + "P2_HILout": { + "mean_abs_err": 0.035880365320508645, + "std_abs_err": 0.5978974388765312 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.5193861634427321, + "std_abs_err": 0.9258294502731639 + }, + "P2_SIT02": { + "mean_abs_err": 0.019901909119880656, + "std_abs_err": 0.6487081324931596 + }, + "P2_VT01": { + "mean_abs_err": 3.983871846457987e-05, + "std_abs_err": 0.00048292196426311107 + }, + "P2_VXT02": { + "mean_abs_err": 0.0008606598899962847, + "std_abs_err": 0.006224256532518291 + }, + "P2_VXT03": { + "mean_abs_err": 0.001263755414677492, + "std_abs_err": 0.007909578018699182 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006738760346043371, + "std_abs_err": 0.005869537952097048 + }, + "P2_VYT03": { + "mean_abs_err": 0.0010588367562194279, + "std_abs_err": 0.009393963692701346 + }, + "P3_FIT01": { + "mean_abs_err": 4.197662930274646, + "std_abs_err": 7.61479856178039 + }, + "P3_LCP01D": { + "mean_abs_err": 9.694041417655171, + "std_abs_err": 11.390745290748782 + }, + "P3_LCV01D": { + "mean_abs_err": 677.0013950541606, + "std_abs_err": 89.09592911473283 + }, + "P3_LIT01": { + "mean_abs_err": 2.360034673156406, + "std_abs_err": 16.887248580934283 + }, + "P3_PIT01": { + "mean_abs_err": 3.4464678100057426, + "std_abs_err": 6.650715677699509 + }, + "P4_HT_FD": { + "mean_abs_err": 0.0010975933046482602, + "std_abs_err": 0.003600711608594502 + }, + "P4_HT_LD": { + "mean_abs_err": 0.23665180052267232, + "std_abs_err": 0.17982503009853446 + }, + "P4_HT_PO": { + "mean_abs_err": 1.2707050437847514, + "std_abs_err": 0.21002688171716954 + }, + "P4_LD": { + "mean_abs_err": 0.004257648361033262, + "std_abs_err": 0.4112773810306152 + }, + "P4_ST_FD": { + "mean_abs_err": 3.349093678351776e-05, + "std_abs_err": 0.002404730111578669 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.611589429565356, + "std_abs_err": 44.86286670571303 + }, + "P4_ST_LD": { + "mean_abs_err": 0.08954652809910613, + "std_abs_err": 0.7405165861296439 + }, + "P4_ST_PO": { + "mean_abs_err": 0.10311497842451445, + "std_abs_err": 0.8233779333265332 + }, + "P4_ST_PT01": { + "mean_abs_err": 5.836923876371657, + "std_abs_err": 10.596449863106596 + }, + "P4_ST_TT01": { + "mean_abs_err": 0.5638297784826136, + "std_abs_err": 0.34213040835190967 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.6595583333333334, + "P1_B2016": 0.043391666666666606, + "P1_B3004": 0.671875, + "P1_B3005": 0.5293, + "P1_B4002": 0.7923208333333334, + "P1_B4005": 0.3062625, + "P1_B400B": 0.46184583333333334, + "P1_B4022": 0.5756333333333334, + "P1_FCV02Z": 0.7746, + "P1_FCV03D": 0.45162500000000005, + "P1_FCV03Z": 0.40985833333333327, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4750083333333333, + "P1_FT02": 0.4545666666666666, + "P1_FT02Z": 0.4029958333333334, + "P1_FT03": 0.5241, + "P1_FT03Z": 0.46950833333333336, + "P1_LCV01D": 0.3859916666666667, + "P1_LCV01Z": 0.397625, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5304583333333333, + "P1_PCV01Z": 0.5184416666666667, + "P1_PCV02Z": 0.6588541666666666, + "P1_PIT01": 0.17675833333333335, + "P1_PIT02": 0.8118208333333334, + "P1_TIT01": 0.16881250000000003, + "P1_TIT02": 0.10090416666666668, + "P2_24Vdc": 0.13548333333333334, + "P2_CO_rpm": 0.0927541666666667, + "P2_HILout": 0.08242499999999997, + "P2_MSD": 0.0, + "P2_SIT01": 0.48742916666666664, + "P2_SIT02": 0.5013166666666666, + "P2_VT01": 0.12336250000000004, + "P2_VXT02": 0.28222916666666664, + "P2_VXT03": 0.43657500000000005, + "P2_VYT02": 0.25022083333333334, + "P2_VYT03": 0.26955833333333334, + "P3_FIT01": 0.0821, + "P3_LCP01D": 0.04134166666666661, + "P3_LCV01D": 0.235225, + "P3_LIT01": 0.09348749999999995, + "P3_PIT01": 0.1158541666666667, + "P4_HT_FD": 0.5712458333333333, + "P4_HT_LD": 0.06742500000000001, + "P4_HT_PO": 0.05970416666666667, + "P4_LD": 0.031329166666666686, + "P4_ST_FD": 0.024483333333333412, + "P4_ST_GOV": 0.035274999999999945, + "P4_ST_LD": 0.03237500000000004, + "P4_ST_PO": 0.03612083333333338, + "P4_ST_PT01": 0.20012916666666664, + "P4_ST_TT01": 0.07880833333333337 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.02845, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0011099999999999999, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.01158999999999999, + "q25_diff": 0.010220000000000118, + "q50_diff": 0.005319999999999991, + "q75_diff": 0.014129999999999976, + "q95_diff": 0.02967999999999993 + }, + "P1_B3004": { + "q05_diff": 14.80313000000001, + "q25_diff": 1.6197799999999916, + "q50_diff": 6.726189999999974, + "q75_diff": 8.508819999999957, + "q95_diff": 37.86444 + }, + "P1_B3005": { + "q05_diff": 40.40789999999993, + "q25_diff": 8.045049999999947, + "q50_diff": 89.51202, + "q75_diff": 104.97393, + "q95_diff": 4.4593499999998585 + }, + "P1_B4002": { + "q05_diff": 0.4564699999999995, + "q25_diff": 1.6555000000000035, + "q50_diff": 1.6555000000000035, + "q75_diff": 0.9287000000000063, + "q95_diff": 0.16969000000000278 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 81.96193, + "q50_diff": 0.04000000000000625, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 2.9795300000000005, + "q25_diff": 2610.38788, + "q50_diff": 189.59229000000005, + "q75_diff": 9.545890000000327, + "q95_diff": 7.019040000000132 + }, + "P1_B4022": { + "q05_diff": 0.38222999999999985, + "q25_diff": 1.3479000000000028, + "q50_diff": 1.4273799999999994, + "q75_diff": 0.9127399999999994, + "q95_diff": 0.5270299999999963 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.05340999999999996, + "q75_diff": 38.169869999999996, + "q95_diff": 0.030520000000009873 + }, + "P1_FCV03D": { + "q05_diff": 1.3654200000000003, + "q25_diff": 0.2554099999999977, + "q50_diff": 8.60333, + "q75_diff": 10.84863, + "q95_diff": 0.824300000000008 + }, + "P1_FCV03Z": { + "q05_diff": 1.7547599999999974, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.148199999999996, + "q75_diff": 10.864249999999998, + "q95_diff": 0.9307899999999876 + }, + "P1_FT01": { + "q05_diff": 8.109370000000013, + "q25_diff": 8.601060000000018, + "q50_diff": 32.806329999999974, + "q75_diff": 37.95618999999999, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 38.15130999999997, + "q25_diff": 18.69965000000002, + "q50_diff": 74.79840000000002, + "q75_diff": 86.54010000000005, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.3814799999999998, + "q25_diff": 1899.23904, + "q50_diff": 258.24926000000005, + "q75_diff": 12.970209999999952, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 1.9862300000000026, + "q25_diff": 2300.54303, + "q50_diff": 14.318849999999657, + "q75_diff": 7.299800000000232, + "q95_diff": 6.317140000000109 + }, + "P1_FT03": { + "q05_diff": 37.54695000000001, + "q25_diff": 1.5381300000000238, + "q50_diff": 51.33557000000002, + "q75_diff": 59.82164, + "q95_diff": 1.9374400000000378 + }, + "P1_FT03Z": { + "q05_diff": 23.41980000000001, + "q25_diff": 2.9805299999999306, + "q50_diff": 84.98266000000012, + "q75_diff": 100.6383699999999, + "q95_diff": 3.5079399999999623 + }, + "P1_LCV01D": { + "q05_diff": 0.17497000000000007, + "q25_diff": 0.29953000000000074, + "q50_diff": 4.171619999999999, + "q75_diff": 3.162939999999999, + "q95_diff": 0.7125800000000009 + }, + "P1_LCV01Z": { + "q05_diff": 0.0, + "q25_diff": 0.5874600000000001, + "q50_diff": 4.638670000000001, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.9536700000000007 + }, + "P1_LIT01": { + "q05_diff": 14.395199999999988, + "q25_diff": 1.0270700000000375, + "q50_diff": 6.185159999999996, + "q75_diff": 8.229829999999993, + "q95_diff": 34.807680000000005 + }, + "P1_PCV01D": { + "q05_diff": 0.18625000000000114, + "q25_diff": 2.7490099999999984, + "q50_diff": 18.274499999999996, + "q75_diff": 43.266600000000004, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 0.22888000000000375, + "q25_diff": 2.0370500000000007, + "q50_diff": 16.029360000000004, + "q75_diff": 43.33495, + "q95_diff": 45.1493 + }, + "P1_PCV02Z": { + "q05_diff": 0.2288800000000002, + "q25_diff": 0.2212499999999995, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_PIT01": { + "q05_diff": 0.035090000000000066, + "q25_diff": 0.016790000000000083, + "q50_diff": 0.006099999999999994, + "q75_diff": 0.02364999999999995, + "q95_diff": 0.008410000000000029 + }, + "P1_PIT02": { + "q05_diff": 0.03433, + "q25_diff": 0.05263999999999999, + "q50_diff": 0.053409999999999985, + "q75_diff": 0.054170000000000024, + "q95_diff": 0.05493000000000001 + }, + "P1_TIT01": { + "q05_diff": 0.07092999999999705, + "q25_diff": 0.07629999999999626, + "q50_diff": 0.15259000000000356, + "q75_diff": 0.3509600000000006, + "q95_diff": 0.01525999999999783 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.062220000000003495, + "q50_diff": 0.10680999999999585, + "q75_diff": 0.2908199999999965, + "q95_diff": 0.4344200000000029 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.00013999999999825263, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.00010999999999938836 + }, + "P2_CO_rpm": { + "q05_diff": 10.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 3.0, + "q95_diff": 0.33202999999775784 + }, + "P2_HILout": { + "q05_diff": 0.3295900000000529, + "q25_diff": 1.6540499999999838, + "q50_diff": 0.31737999999995736, + "q75_diff": 1.7334000000000742, + "q95_diff": 2.2399900000000343 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 2.0, + "q25_diff": 3.0, + "q50_diff": 4.5, + "q75_diff": 6.0, + "q95_diff": 6.0 + }, + "P2_SIT02": { + "q05_diff": 2.6317200000000867, + "q25_diff": 3.570310000000063, + "q50_diff": 5.6193300000001045, + "q75_diff": 6.361210000000028, + "q95_diff": 6.843689999999924 + }, + "P2_VT01": { + "q05_diff": 0.008480000000000487, + "q25_diff": 0.015229999999998967, + "q50_diff": 0.014120000000000132, + "q75_diff": 0.013400000000000745, + "q95_diff": 0.01904000000000039 + }, + "P2_VXT02": { + "q05_diff": 0.12995, + "q25_diff": 0.29942, + "q50_diff": 0.2857599999999998, + "q75_diff": 0.26654, + "q95_diff": 0.45904999999999996 + }, + "P2_VXT03": { + "q05_diff": 0.10075999999999996, + "q25_diff": 0.2691999999999999, + "q50_diff": 0.2973199999999998, + "q75_diff": 0.32702999999999993, + "q95_diff": 0.61067 + }, + "P2_VYT02": { + "q05_diff": 0.09643999999999986, + "q25_diff": 0.26202000000000014, + "q50_diff": 0.22612000000000032, + "q75_diff": 0.2354699999999994, + "q95_diff": 0.3892100000000003 + }, + "P2_VYT03": { + "q05_diff": 0.055449999999999555, + "q25_diff": 0.1862699999999995, + "q50_diff": 0.17562000000000033, + "q75_diff": 0.18233999999999995, + "q95_diff": 0.35440000000000005 + }, + "P3_FIT01": { + "q05_diff": 1.0, + "q25_diff": 0.0, + "q50_diff": 48.63019, + "q75_diff": 81.7773400000001, + "q95_diff": 18.52930000000015 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 95.69885, + "q75_diff": 133.79297000000042, + "q95_diff": 240.0 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 320.0, + "q50_diff": 3189.4414100000004, + "q75_diff": 175.13378999999986, + "q95_diff": 23.926760000000286 + }, + "P3_LIT01": { + "q05_diff": 1135.7871100000002, + "q25_diff": 26.398440000000846, + "q50_diff": 650.5126999999993, + "q75_diff": 1148.9355499999983, + "q95_diff": 1.3242199999986042 + }, + "P3_PIT01": { + "q05_diff": 1.0, + "q25_diff": 0.0, + "q50_diff": 1.0, + "q75_diff": 79.31182999999999, + "q95_diff": 62.41211000000021 + }, + "P4_HT_FD": { + "q05_diff": 0.00174, + "q25_diff": 0.00043000000000000004, + "q50_diff": 0.0005700000000000001, + "q75_diff": 7e-05, + "q95_diff": 0.0005099999999999999 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 27.14122, + "q75_diff": 0.13020999999999106, + "q95_diff": 0.21701000000000192 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 0.9223299999999999, + "q50_diff": 8.89752, + "q75_diff": 0.054199999999994475, + "q95_diff": 0.6510700000000043 + }, + "P4_LD": { + "q05_diff": 0.5629900000000134, + "q25_diff": 3.0934100000000058, + "q50_diff": 3.561399999999992, + "q75_diff": 0.026119999999991705, + "q95_diff": 4.2164600000000405 + }, + "P4_ST_FD": { + "q05_diff": 2.9999999999999645e-05, + "q25_diff": 0.0, + "q50_diff": 7.999999999999999e-05, + "q75_diff": 7.000000000000008e-05, + "q95_diff": 0.00014999999999999996 + }, + "P4_ST_GOV": { + "q05_diff": 128.33691, + "q25_diff": 38.02538999999888, + "q50_diff": 32.49022999999943, + "q75_diff": 105.10155999999915, + "q95_diff": 414.03515999999945 + }, + "P4_ST_LD": { + "q05_diff": 0.7499099999999999, + "q25_diff": 0.48076999999994996, + "q50_diff": 0.38602000000003045, + "q75_diff": 2.0497100000000046, + "q95_diff": 7.9558399999999665 + }, + "P4_ST_PO": { + "q05_diff": 2.1662000000000035, + "q25_diff": 0.725529999999992, + "q50_diff": 0.6129799999999932, + "q75_diff": 2.143250000000023, + "q95_diff": 7.837030000000027 + }, + "P4_ST_PT01": { + "q05_diff": 4.0, + "q25_diff": 14.0, + "q50_diff": 2.0, + "q75_diff": 0.0, + "q95_diff": 30.0 + }, + "P4_ST_TT01": { + "q05_diff": 6.0, + "q25_diff": 6.0, + "q50_diff": 3.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 1.0624831607922858, + "P1_B2016": 0.2561068038657559, + "P1_B3004": 0.9420514424699565, + "P1_B3005": 0.9755864990985265, + "P1_B4002": 0.9939428775878422, + "P1_B4005": 1.0901635813805555, + "P1_B400B": 1.0636685342327254, + "P1_B4022": 1.0728287547128275, + "P1_FCV02Z": 1.0856923346443097, + "P1_FCV03D": 1.0287616800381814, + "P1_FCV03Z": 1.01920928123416, + "P1_FT01": 1.0837129234395464, + "P1_FT01Z": 0.8647923079278955, + "P1_FT02": 1.0102725969371698, + "P1_FT02Z": 1.0572315062387077, + "P1_FT03": 0.8468517424320046, + "P1_FT03Z": 0.9861265787637634, + "P1_LCV01D": 0.9922897795117445, + "P1_LCV01Z": 0.9035160861736835, + "P1_LIT01": 0.963461971457682, + "P1_PCV01D": 0.9847169107083952, + "P1_PCV01Z": 1.1483081333334428, + "P1_PCV02Z": 0.5764046458797007, + "P1_PIT01": 0.4203947521554928, + "P1_PIT02": 0.9828387650450174, + "P1_TIT01": 0.7798971257915577, + "P1_TIT02": 0.865590207042142, + "P2_24Vdc": 0.039965792976989475, + "P2_CO_rpm": 0.27366047721795406, + "P2_HILout": 0.24706059027478758, + "P2_MSD": 0.0, + "P2_SIT01": 0.6624061266276653, + "P2_SIT02": 0.625353002969849, + "P2_VT01": 0.44589844834894954, + "P2_VXT02": 0.024527633833306917, + "P2_VXT03": 0.26813324104938296, + "P2_VYT02": 0.5193038198965017, + "P2_VYT03": 0.39748846203270954, + "P3_FIT01": 0.8757503767058722, + "P3_LCP01D": 0.956442423364842, + "P3_LCV01D": 0.845280051194023, + "P3_LIT01": 0.9604496464042352, + "P3_PIT01": 0.9121126092062325, + "P4_HT_FD": 0.33671422249468896, + "P4_HT_LD": 1.052071148480924, + "P4_HT_PO": 1.0918064404320895, + "P4_LD": 0.8941496210667775, + "P4_ST_FD": 0.29904099302570664, + "P4_ST_GOV": 0.8465337816178481, + "P4_ST_LD": 0.8603354377847401, + "P4_ST_PO": 0.7944502897383621, + "P4_ST_PT01": 0.9795661975361911, + "P4_ST_TT01": 0.8779218332437576 + }, + "discrete_jsd": { + "P1_FCV01D": 0.09833180131798225, + "P1_FCV01Z": 0.6325772640315827, + "P1_FCV02D": 0.07925115505562264, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_type_routing__seed1337/comprehensive_eval.json b/example/results/ablations/runs/no_type_routing__seed1337/comprehensive_eval.json new file mode 100644 index 0000000..85524a9 --- /dev/null +++ b/example/results/ablations/runs/no_type_routing__seed1337/comprehensive_eval.json @@ -0,0 +1,168 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_type_routing__seed1337/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.35632916666666664, + "avg_jsd": 0.021975792751969963, + "avg_lag1_diff": 0.13777518542478578 + }, + "two_sample": { + "continuous_mmd_rbf": 0.2909727096557617, + "continuous_mmd_gamma": 4.0559094442168274e-05, + "histogram_mmd_rbf": 0.2910001277923584, + "histogram_mmd_gamma": 4.055096083600875e-05, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 108.9692611694336, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 1.9749261536536356, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.3238775134086609, + "corr_frobenius": 22.76103973388672, + "lag1_corr_mean_abs_diff": 0.3250409662723541, + "lag1_corr_frobenius": 22.80216407775879, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.42320194840431213, + "corr_frobenius": 14.319388389587402, + "lag1_corr_mean_abs_diff": 0.43265166878700256, + "lag1_corr_frobenius": 14.421116828918457 + }, + "P2": { + "corr_mean_abs_diff": 0.12742209434509277, + "corr_frobenius": 2.246002435684204, + "lag1_corr_mean_abs_diff": 0.1239493265748024, + "lag1_corr_frobenius": 2.167053461074829 + }, + "P3": { + "corr_mean_abs_diff": 0.5636910796165466, + "corr_frobenius": 3.4943292140960693, + "lag1_corr_mean_abs_diff": 0.5967755913734436, + "lag1_corr_frobenius": 3.5241942405700684 + }, + "P4": { + "corr_mean_abs_diff": 0.22367851436138153, + "corr_frobenius": 3.3961873054504395, + "lag1_corr_mean_abs_diff": 0.22827841341495514, + "lag1_corr_frobenius": 3.391732931137085 + } + } + }, + "frequency": { + "avg_psd_l1": 0.01135118305683136, + "avg_psd_cosine": 0.14178049901746353, + "avg_low_high_ratio_abs_diff": 14.099956321555485 + }, + "type_metrics": { + "type1_program": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + }, + "type2_controller": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + }, + "type3_actuator": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + }, + "type4_pv": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + }, + "type5_program_proxy": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + }, + "type6_aux": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 1.017059564590454, + "mae": 0.8145182728767395 + }, + "real_plus_synthetic": { + "rmse": 0.547598659992218, + "mae": 0.3571276366710663 + }, + "rmse_ratio_synth_to_real": 1.821541842103409 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8252551020408163, + "auprc": 0.5865288947855654, + "threshold": 0.003061630530282855, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6756756756756758, + "best_f1_threshold": 1.4288572072982788 + }, + "real_plus_synthetic": { + "auroc": 0.8795222634508348, + "auprc": 0.6465701950949686, + "threshold": 0.29103735089302063, + "f1": 0.2397003745318352, + "precision": 0.14349775784753363, + "recall": 0.7272727272727273, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.4753831624984741 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_type_routing__seed1337/comprehensive_eval_post.json b/example/results/ablations/runs/no_type_routing__seed1337/comprehensive_eval_post.json new file mode 100644 index 0000000..1c20706 --- /dev/null +++ b/example/results/ablations/runs/no_type_routing__seed1337/comprehensive_eval_post.json @@ -0,0 +1,168 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/ablations/runs/no_type_routing__seed1337/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.2909727096557617, + "continuous_mmd_gamma": 4.0559094442168274e-05, + "histogram_mmd_rbf": 0.2910001277923584, + "histogram_mmd_gamma": 4.055096083600875e-05, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 108.9692611694336, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 1.9749261536536356, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.3238775134086609, + "corr_frobenius": 22.76103973388672, + "lag1_corr_mean_abs_diff": 0.3250409662723541, + "lag1_corr_frobenius": 22.80216407775879, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.42320194840431213, + "corr_frobenius": 14.319388389587402, + "lag1_corr_mean_abs_diff": 0.43265166878700256, + "lag1_corr_frobenius": 14.421116828918457 + }, + "P2": { + "corr_mean_abs_diff": 0.12742209434509277, + "corr_frobenius": 2.246002435684204, + "lag1_corr_mean_abs_diff": 0.1239493265748024, + "lag1_corr_frobenius": 2.167053461074829 + }, + "P3": { + "corr_mean_abs_diff": 0.5636910796165466, + "corr_frobenius": 3.4943292140960693, + "lag1_corr_mean_abs_diff": 0.5967755913734436, + "lag1_corr_frobenius": 3.5241942405700684 + }, + "P4": { + "corr_mean_abs_diff": 0.22367851436138153, + "corr_frobenius": 3.3961873054504395, + "lag1_corr_mean_abs_diff": 0.22827841341495514, + "lag1_corr_frobenius": 3.391732931137085 + } + } + }, + "frequency": { + "avg_psd_l1": 0.01135118305683136, + "avg_psd_cosine": 0.14178049901746353, + "avg_low_high_ratio_abs_diff": 14.099956321555485 + }, + "type_metrics": { + "type1_program": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + }, + "type2_controller": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + }, + "type3_actuator": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + }, + "type4_pv": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + }, + "type5_program_proxy": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + }, + "type6_aux": { + "features": [], + "generated": {}, + "reference": {}, + "aggregates": {} + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 1.017059564590454, + "mae": 0.8145182728767395 + }, + "real_plus_synthetic": { + "rmse": 0.547598659992218, + "mae": 0.3571276366710663 + }, + "rmse_ratio_synth_to_real": 1.821541842103409 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8252551020408163, + "auprc": 0.5865288947855654, + "threshold": 0.003061630530282855, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6756756756756758, + "best_f1_threshold": 1.4288572072982788 + }, + "real_plus_synthetic": { + "auroc": 0.8795222634508348, + "auprc": 0.6465701950949686, + "threshold": 0.29103735089302063, + "f1": 0.2397003745318352, + "precision": 0.14349775784753363, + "recall": 0.7272727272727273, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.4753831624984741 + } + } +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_type_routing__seed1337/eval.json b/example/results/ablations/runs/no_type_routing__seed1337/eval.json new file mode 100644 index 0000000..259df35 --- /dev/null +++ b/example/results/ablations/runs/no_type_routing__seed1337/eval.json @@ -0,0 +1,974 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09193078124999997, + "std": 0.019815537746997347 + }, + "P1_B2016": { + "mean": 1.3915223437499995, + "std": 0.1393654531785485 + }, + "P1_B3004": { + "mean": 399.6991551822917, + "std": 12.518914842422626 + }, + "P1_B3005": { + "mean": 1051.685153828123, + "std": 63.59243548494965 + }, + "P1_B4002": { + "mean": 32.006918411458315, + "std": 0.5813814817344444 + }, + "P1_B4005": { + "mean": 47.9263465885417, + "std": 49.790997850542055 + }, + "P1_B400B": { + "mean": 1743.4152112500008, + "std": 1229.6701768640178 + }, + "P1_B4022": { + "mean": 36.06212914062498, + "std": 0.6831827635704265 + }, + "P1_FCV02Z": { + "mean": 18.558006718750008, + "std": 32.538737586534346 + }, + "P1_FCV03D": { + "mean": 58.570887838541644, + "std": 6.6406258388712 + }, + "P1_FCV03Z": { + "mean": 59.17428062499997, + "std": 6.677133912549545 + }, + "P1_FT01": { + "mean": 194.80419330729157, + "std": 32.02466301288328 + }, + "P1_FT01Z": { + "mean": 877.2179718489588, + "std": 84.5261318996543 + }, + "P1_FT02": { + "mean": 1128.853577473959, + "std": 894.9150736225375 + }, + "P1_FT02Z": { + "mean": 1743.4468020052082, + "std": 1231.48016631041 + }, + "P1_FT03": { + "mean": 282.4489581510417, + "std": 33.658513522739625 + }, + "P1_FT03Z": { + "mean": 1060.5806286718764, + "std": 61.63984648937954 + }, + "P1_LCV01D": { + "mean": 12.16772833333333, + "std": 3.046298692652483 + }, + "P1_LCV01Z": { + "mean": 11.847819270833337, + "std": 3.1165923387339785 + }, + "P1_LIT01": { + "mean": 403.9420300520833, + "std": 15.781781361473751 + }, + "P1_PCV01D": { + "mean": 67.44987505208331, + "std": 23.854386093147486 + }, + "P1_PCV01Z": { + "mean": 68.38738161458328, + "std": 23.947728580011706 + }, + "P1_PCV02Z": { + "mean": 11.781131171874993, + "std": 0.010866175171277234 + }, + "P1_PIT01": { + "mean": 1.3746076302083339, + "std": 0.05960672856037052 + }, + "P1_PIT02": { + "mean": 0.33484200520833357, + "std": 0.1948743524939526 + }, + "P1_TIT01": { + "mean": 35.63990026041668, + "std": 0.5343657077642556 + }, + "P1_TIT02": { + "mean": 36.52362546874998, + "std": 1.230905429705141 + }, + "P2_24Vdc": { + "mean": 28.02859052083333, + "std": 0.003421123788857911 + }, + "P2_CO_rpm": { + "mean": 54102.357055650995, + "std": 23.98807742907888 + }, + "P2_HILout": { + "mean": 711.4852836458331, + "std": 9.41984148144893 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.4609375000005, + "std": 5.351219099895989 + }, + "P2_SIT02": { + "mean": 783.9247166145833, + "std": 5.839942048250984 + }, + "P2_VT01": { + "mean": 11.923782994791672, + "std": 0.06898982316928673 + }, + "P2_VXT02": { + "mean": -3.299330755208332, + "std": 0.47730137668575 + }, + "P2_VXT03": { + "mean": -1.3056674739583336, + "std": 0.35739736340274214 + }, + "P2_VYT02": { + "mean": 3.9808523437499996, + "std": 0.5057055033410006 + }, + "P2_VYT03": { + "mean": 6.233328177083334, + "std": 0.38944681565165135 + }, + "P3_FIT01": { + "mean": 426.19300627604184, + "std": 1326.939025355576 + }, + "P3_LCP01D": { + "mean": 4590.694557578123, + "std": 5143.367197123021 + }, + "P3_LCV01D": { + "mean": 7119.145154817703, + "std": 6880.3312187991705 + }, + "P3_LIT01": { + "mean": 13352.39418132812, + "std": 4371.087325465592 + }, + "P3_PIT01": { + "mean": -18.901041666666664, + "std": 1.4544786180892033 + }, + "P4_HT_FD": { + "mean": -0.003472005208333333, + "std": 0.012897029770787882 + }, + "P4_HT_LD": { + "mean": 23.069752760416677, + "std": 33.75028540477662 + }, + "P4_HT_PO": { + "mean": 33.93330171874999, + "std": 31.615734310851927 + }, + "P4_LD": { + "mean": 366.1256761718753, + "std": 61.02547609796988 + }, + "P4_ST_FD": { + "mean": -7.067708333333341e-05, + "std": 0.004046225660922525 + }, + "P4_ST_GOV": { + "mean": 17918.27675395833, + "std": 1895.9205860154618 + }, + "P4_ST_LD": { + "mean": 332.0906425260417, + "std": 38.58216792421147 + }, + "P4_ST_PO": { + "mean": 332.3652670833332, + "std": 35.25397160583083 + }, + "P4_ST_PT01": { + "mean": 10025.982460026047, + "std": 29.990919766020852 + }, + "P4_ST_TT01": { + "mean": 27606.179697682306, + "std": 25.153659557189588 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.0016368708409757043, + "std_abs_err": 0.0018171006649049422 + }, + "P1_B2016": { + "mean_abs_err": 0.00025601908523253947, + "std_abs_err": 0.0035918907019854285 + }, + "P1_B3004": { + "mean_abs_err": 3.686908642234471, + "std_abs_err": 2.899234745619184 + }, + "P1_B3005": { + "mean_abs_err": 9.363632700506741, + "std_abs_err": 2.4091189739493686 + }, + "P1_B4002": { + "mean_abs_err": 0.3100079084215679, + "std_abs_err": 0.0027079825353665266 + }, + "P1_B4005": { + "mean_abs_err": 12.598760427656224, + "std_abs_err": 7.609700402832367 + }, + "P1_B400B": { + "mean_abs_err": 35.060658927967324, + "std_abs_err": 36.3179039449451 + }, + "P1_B4022": { + "mean_abs_err": 0.0009544932224585523, + "std_abs_err": 0.011731908735608143 + }, + "P1_FCV02Z": { + "mean_abs_err": 6.392928181722269, + "std_abs_err": 7.899747852723834 + }, + "P1_FCV03D": { + "mean_abs_err": 0.003024387694814834, + "std_abs_err": 0.041130120162445216 + }, + "P1_FCV03Z": { + "mean_abs_err": 0.12475855518986378, + "std_abs_err": 0.03767005178205718 + }, + "P1_FT01": { + "mean_abs_err": 0.08053895101130593, + "std_abs_err": 4.676681174238954 + }, + "P1_FT01Z": { + "mean_abs_err": 1.8317096551937766, + "std_abs_err": 8.760428936336055 + }, + "P1_FT02": { + "mean_abs_err": 3.9645757606672305, + "std_abs_err": 5.718909771124913 + }, + "P1_FT02Z": { + "mean_abs_err": 35.031644603440554, + "std_abs_err": 38.13481053926034 + }, + "P1_FT03": { + "mean_abs_err": 0.19001212518719512, + "std_abs_err": 0.18430041144412002 + }, + "P1_FT03Z": { + "mean_abs_err": 0.4614096766451894, + "std_abs_err": 0.5143576257622158 + }, + "P1_LCV01D": { + "mean_abs_err": 0.0133956715581931, + "std_abs_err": 0.1316837396223618 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.11825475850458034, + "std_abs_err": 0.15312284138866694 + }, + "P1_LIT01": { + "mean_abs_err": 0.17284211092299984, + "std_abs_err": 0.14988331309977987 + }, + "P1_PCV01D": { + "mean_abs_err": 0.015698683608064812, + "std_abs_err": 0.08461939224191184 + }, + "P1_PCV01Z": { + "mean_abs_err": 0.020770182319310493, + "std_abs_err": 0.08312876934806113 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.06162177472993413, + "std_abs_err": 0.08597829029337013 + }, + "P1_PIT01": { + "mean_abs_err": 0.002873590545058091, + "std_abs_err": 0.0018258308095115833 + }, + "P1_PIT02": { + "mean_abs_err": 0.15231508513318315, + "std_abs_err": 0.30383728556092726 + }, + "P1_TIT01": { + "mean_abs_err": 0.04130798201653363, + "std_abs_err": 0.04891491942169335 + }, + "P1_TIT02": { + "mean_abs_err": 0.0013532095865542715, + "std_abs_err": 0.009146703680877888 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00011918179433934029, + "std_abs_err": 7.865767445099613e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.3531422316809767, + "std_abs_err": 1.5323456542905163 + }, + "P2_HILout": { + "mean_abs_err": 0.022474375736919683, + "std_abs_err": 0.9651156041927678 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.44297991344251386, + "std_abs_err": 0.15230373945176368 + }, + "P2_SIT02": { + "mean_abs_err": 0.02097873203706513, + "std_abs_err": 0.6493622914380177 + }, + "P2_VT01": { + "mean_abs_err": 3.265121846141028e-05, + "std_abs_err": 0.0004761410972846658 + }, + "P2_VXT02": { + "mean_abs_err": 0.0008477171816640983, + "std_abs_err": 0.006268450414529725 + }, + "P2_VXT03": { + "mean_abs_err": 0.0012258127063442537, + "std_abs_err": 0.007769984239088479 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006165062429372981, + "std_abs_err": 0.00579505034821276 + }, + "P2_VYT03": { + "mean_abs_err": 0.0010769357145505865, + "std_abs_err": 0.009643853680559755 + }, + "P3_FIT01": { + "mean_abs_err": 724.7071698301413, + "std_abs_err": 452.7856724346989 + }, + "P3_LCP01D": { + "mean_abs_err": 5.9674093375488155, + "std_abs_err": 13.367199407141925 + }, + "P3_LCV01D": { + "mean_abs_err": 14.73156216353891, + "std_abs_err": 40.403322983349426 + }, + "P3_LIT01": { + "mean_abs_err": 2.4503113658665825, + "std_abs_err": 16.912688537655413 + }, + "P3_PIT01": { + "mean_abs_err": 668.2589446899943, + "std_abs_err": 1149.467452089106 + }, + "P4_HT_FD": { + "mean_abs_err": 0.003368817262981593, + "std_abs_err": 0.010865011456383782 + }, + "P4_HT_LD": { + "mean_abs_err": 10.925164319268937, + "std_abs_err": 0.3264918947783144 + }, + "P4_HT_PO": { + "mean_abs_err": 0.05050607600691848, + "std_abs_err": 0.17486309353045115 + }, + "P4_LD": { + "mean_abs_err": 0.0030776223190400742, + "std_abs_err": 0.4096697285268718 + }, + "P4_ST_FD": { + "mean_abs_err": 4.1563853450184596e-05, + "std_abs_err": 0.0024040494029324956 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.315492710804392, + "std_abs_err": 43.77737002768845 + }, + "P4_ST_LD": { + "mean_abs_err": 0.08774535622399071, + "std_abs_err": 0.7289986325939211 + }, + "P4_ST_PO": { + "mean_abs_err": 0.09629575967443316, + "std_abs_err": 0.8030281034225268 + }, + "P4_ST_PT01": { + "mean_abs_err": 24.59269679303543, + "std_abs_err": 12.206149740701722 + }, + "P4_ST_TT01": { + "mean_abs_err": 2.852993997224985, + "std_abs_err": 1.2363377526212602 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.701225, + "P1_B2016": 0.04325000000000001, + "P1_B3004": 0.6640625, + "P1_B3005": 0.5657583333333334, + "P1_B4002": 0.81055, + "P1_B4005": 0.42952916666666663, + "P1_B400B": 0.45285833333333336, + "P1_B4022": 0.6185416666666667, + "P1_FCV02Z": 0.7563708333333333, + "P1_FCV03D": 0.4882875, + "P1_FCV03Z": 0.4923916666666667, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4750083333333333, + "P1_FT02": 0.4530583333333334, + "P1_FT02Z": 0.4536416666666666, + "P1_FT03": 0.5244541666666667, + "P1_FT03Z": 0.5245541666666667, + "P1_LCV01D": 0.4472333333333333, + "P1_LCV01Z": 0.4523125, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5577125, + "P1_PCV01Z": 0.5588041666666668, + "P1_PCV02Z": 1.0, + "P1_PIT01": 0.1511041666666666, + "P1_PIT02": 0.6707083333333334, + "P1_TIT01": 0.17708749999999995, + "P1_TIT02": 0.10090833333333338, + "P2_24Vdc": 0.11985833333333334, + "P2_CO_rpm": 0.09294999999999998, + "P2_HILout": 0.0826, + "P2_MSD": 0.0, + "P2_SIT01": 0.4645208333333333, + "P2_SIT02": 0.5016666666666667, + "P2_VT01": 0.12326250000000005, + "P2_VXT02": 0.2816291666666666, + "P2_VXT03": 0.43657500000000005, + "P2_VYT02": 0.25022083333333334, + "P2_VYT03": 0.26955416666666665, + "P3_FIT01": 0.38516249999999996, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.172725, + "P3_LIT01": 0.09314166666666668, + "P3_PIT01": 0.43189999999999995, + "P4_HT_FD": 0.2294, + "P4_HT_LD": 0.4056375, + "P4_HT_PO": 0.1821708333333333, + "P4_LD": 0.031329166666666686, + "P4_ST_FD": 0.03490000000000004, + "P4_ST_GOV": 0.03517083333333337, + "P4_ST_LD": 0.032299999999999995, + "P4_ST_PO": 0.03607499999999997, + "P4_ST_PT01": 0.4944916666666666, + "P4_ST_TT01": 0.1453625 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.00022999999999999757, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0015000000000000013, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.01200000000000001, + "q25_diff": 0.010310000000000041, + "q50_diff": 0.005179999999999962, + "q75_diff": 0.014189999999999925, + "q95_diff": 0.030819999999999848 + }, + "P1_B3004": { + "q05_diff": 17.269769999999994, + "q25_diff": 1.658420000000035, + "q50_diff": 6.61248999999998, + "q75_diff": 8.571069999999963, + "q95_diff": 18.877499999999998 + }, + "P1_B3005": { + "q05_diff": 92.21636999999998, + "q25_diff": 3.1389700000000857, + "q50_diff": 89.51202, + "q75_diff": 103.22381000000007, + "q95_diff": 4.042119999999841 + }, + "P1_B4002": { + "q05_diff": 0.5865699999999983, + "q25_diff": 2.242070000000002, + "q50_diff": 1.6555000000000035, + "q75_diff": 1.118700000000004, + "q95_diff": 0.19520000000000692 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 100.0, + "q50_diff": 100.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2526.96808, + "q50_diff": 191.70996999999988, + "q75_diff": 9.545890000000327, + "q95_diff": 7.019040000000132 + }, + "P1_B4022": { + "q05_diff": 0.36859000000000464, + "q25_diff": 1.335410000000003, + "q50_diff": 1.4134700000000038, + "q75_diff": 1.1175700000000006, + "q95_diff": 0.4899300000000011 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.07630000000000003, + "q75_diff": 43.01453, + "q95_diff": 0.022880000000000678 + }, + "P1_FCV03D": { + "q05_diff": 2.89734, + "q25_diff": 0.09548000000000201, + "q50_diff": 8.54806, + "q75_diff": 11.147739999999999, + "q95_diff": 0.6383199999999931 + }, + "P1_FCV03Z": { + "q05_diff": 3.2806399999999982, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.96454, + "q75_diff": 11.337280000000007, + "q95_diff": 0.4730199999999911 + }, + "P1_FT01": { + "q05_diff": 7.82011, + "q25_diff": 8.588690000000014, + "q50_diff": 32.04345999999998, + "q75_diff": 37.76549, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 39.17327999999998, + "q25_diff": 19.58227999999997, + "q50_diff": 74.36358000000007, + "q75_diff": 86.10528, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.5722499999999995, + "q25_diff": 1935.76794, + "q50_diff": 268.60411999999997, + "q75_diff": 12.970209999999952, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 8.937850000000001, + "q25_diff": 2770.27588, + "q50_diff": 189.2133799999997, + "q75_diff": 9.545890000000327, + "q95_diff": 7.019040000000132 + }, + "P1_FT03": { + "q05_diff": 36.04084, + "q25_diff": 0.953630000000004, + "q50_diff": 51.52633000000003, + "q75_diff": 59.761000000000024, + "q95_diff": 1.555970000000002 + }, + "P1_FT03Z": { + "q05_diff": 81.72820999999999, + "q25_diff": 2.672670000000039, + "q50_diff": 87.85290000000009, + "q75_diff": 100.88019000000008, + "q95_diff": 2.282600000000002 + }, + "P1_LCV01D": { + "q05_diff": 0.1012599999999999, + "q25_diff": 1.2061399999999995, + "q50_diff": 4.635400000000001, + "q75_diff": 3.3114999999999988, + "q95_diff": 0.61768 + }, + "P1_LCV01Z": { + "q05_diff": 0.2136199999999997, + "q25_diff": 1.1901799999999998, + "q50_diff": 4.76075, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.7136000000000013 + }, + "P1_LIT01": { + "q05_diff": 14.721709999999973, + "q25_diff": 1.0259100000000103, + "q50_diff": 6.23626999999999, + "q75_diff": 8.178740000000005, + "q95_diff": 34.780489999999986 + }, + "P1_PCV01D": { + "q05_diff": 1.0988299999999995, + "q25_diff": 3.711590000000001, + "q50_diff": 22.252399999999994, + "q75_diff": 44.83375, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 1.0054599999999994, + "q25_diff": 3.814699999999995, + "q50_diff": 22.764490000000002, + "q75_diff": 45.46356000000001, + "q95_diff": 45.1455 + }, + "P1_PCV02Z": { + "q05_diff": 0.2441399999999998, + "q25_diff": 0.2441399999999998, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.2288800000000002, + "q95_diff": 0.2288800000000002 + }, + "P1_PIT01": { + "q05_diff": 0.03492000000000006, + "q25_diff": 0.017550000000000177, + "q50_diff": 0.0045800000000000285, + "q75_diff": 0.022119999999999918, + "q95_diff": 0.019830000000000014 + }, + "P1_PIT02": { + "q05_diff": 0.03051999999999999, + "q25_diff": 0.0015199999999999658, + "q50_diff": 0.0022899999999999587, + "q75_diff": 0.003049999999999997, + "q95_diff": 0.46234 + }, + "P1_TIT01": { + "q05_diff": 0.07628000000000412, + "q25_diff": 0.07629999999999626, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.2593999999999994, + "q95_diff": 0.27465999999999724 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.0614400000000046, + "q50_diff": 0.09154999999999802, + "q75_diff": 0.29144999999999754, + "q95_diff": 0.43856999999999857 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.0008799999999986596, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.000920000000000698 + }, + "P2_CO_rpm": { + "q05_diff": 10.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 2.0, + "q95_diff": 1.0 + }, + "P2_HILout": { + "q05_diff": 0.5802699999999277, + "q25_diff": 2.0135499999998956, + "q50_diff": 0.08672999999998865, + "q75_diff": 1.5440100000000712, + "q95_diff": 1.1977000000000544 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 3.0, + "q25_diff": 3.0, + "q50_diff": 5.0, + "q75_diff": 6.0, + "q95_diff": 6.0 + }, + "P2_SIT02": { + "q05_diff": 2.660160000000019, + "q25_diff": 3.5689700000000357, + "q50_diff": 5.618170000000077, + "q75_diff": 6.357729999999947, + "q95_diff": 6.8461300000000165 + }, + "P2_VT01": { + "q05_diff": 0.00835000000000008, + "q25_diff": 0.015249999999999986, + "q50_diff": 0.014160000000000394, + "q75_diff": 0.01328999999999958, + "q95_diff": 0.018950000000000244 + }, + "P2_VXT02": { + "q05_diff": 0.13046000000000024, + "q25_diff": 0.2993999999999999, + "q50_diff": 0.28474999999999984, + "q75_diff": 0.26602000000000015, + "q95_diff": 0.4596100000000001 + }, + "P2_VXT03": { + "q05_diff": 0.10050999999999988, + "q25_diff": 0.2691699999999999, + "q50_diff": 0.29742, + "q75_diff": 0.32671000000000006, + "q95_diff": 0.61104 + }, + "P2_VYT02": { + "q05_diff": 0.09780999999999995, + "q25_diff": 0.26202000000000014, + "q50_diff": 0.2266300000000001, + "q75_diff": 0.23521999999999998, + "q95_diff": 0.3870300000000002 + }, + "P2_VYT03": { + "q05_diff": 0.05653000000000041, + "q25_diff": 0.18581000000000003, + "q50_diff": 0.17598999999999965, + "q75_diff": 0.18259999999999987, + "q95_diff": 0.3547899999999995 + }, + "P3_FIT01": { + "q05_diff": 1.0, + "q25_diff": 0.0, + "q50_diff": 72.0, + "q75_diff": 2582.0, + "q95_diff": 331.0 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 102.91283999999996, + "q75_diff": 120.49706999999944, + "q95_diff": 9.175779999999577 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 256.0, + "q50_diff": 3238.2841799999997, + "q75_diff": 168.74316, + "q95_diff": 30.367190000000846 + }, + "P3_LIT01": { + "q05_diff": 1135.8911099999996, + "q25_diff": 26.31151999999929, + "q50_diff": 649.4033199999994, + "q75_diff": 1140.2734400000008, + "q95_diff": 2.3339799999994284 + }, + "P3_PIT01": { + "q05_diff": 2.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 928.0, + "q95_diff": 3524.0 + }, + "P4_HT_FD": { + "q05_diff": 0.048029999999999996, + "q25_diff": 0.00014, + "q50_diff": 7.000000000000002e-05, + "q75_diff": 7e-05, + "q95_diff": 0.00021999999999999993 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 54.74537, + "q75_diff": 0.5989600000000053, + "q95_diff": 0.39124999999999943 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 1.33825, + "q50_diff": 14.777420000000003, + "q75_diff": 0.785899999999998, + "q95_diff": 0.09572000000000003 + }, + "P4_LD": { + "q05_diff": 0.5577400000000239, + "q25_diff": 2.9282499999999914, + "q50_diff": 3.4519900000000234, + "q75_diff": 0.16329999999999245, + "q95_diff": 4.151920000000018 + }, + "P4_ST_FD": { + "q05_diff": 2.9999999999999645e-05, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.00014000000000000004, + "q95_diff": 8.000000000000021e-05 + }, + "P4_ST_GOV": { + "q05_diff": 119.87890999999945, + "q25_diff": 38.54102000000057, + "q50_diff": 31.49022999999943, + "q75_diff": 109.1992199999986, + "q95_diff": 417.9902299999994 + }, + "P4_ST_LD": { + "q05_diff": 0.7481400000000349, + "q25_diff": 0.41091999999997597, + "q50_diff": 0.47870000000000346, + "q75_diff": 1.9949300000000108, + "q95_diff": 7.954279999999983 + }, + "P4_ST_PO": { + "q05_diff": 2.229069999999979, + "q25_diff": 0.732910000000004, + "q50_diff": 0.5704400000000192, + "q75_diff": 2.1875, + "q95_diff": 7.791440000000023 + }, + "P4_ST_PT01": { + "q05_diff": 23.0, + "q25_diff": 31.0, + "q50_diff": 54.0, + "q75_diff": 0.0, + "q95_diff": 18.0 + }, + "P4_ST_TT01": { + "q05_diff": 5.0, + "q25_diff": 3.0, + "q50_diff": 4.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 0.0842454873511631, + "P1_B2016": 0.031142256345468855, + "P1_B3004": 0.09059121934513292, + "P1_B3005": 0.10933425266689745, + "P1_B4002": 0.1381657367610738, + "P1_B4005": 0.044394423231015945, + "P1_B400B": 0.2302252103627388, + "P1_B4022": 0.02288387249075763, + "P1_FCV02Z": 0.1551545855219576, + "P1_FCV03D": 0.020915544759740334, + "P1_FCV03Z": 0.03143040195289626, + "P1_FT01": 0.2608931050318283, + "P1_FT01Z": 0.22837148157422993, + "P1_FT02": 0.46676963996583987, + "P1_FT02Z": 0.21179817615122698, + "P1_FT03": 0.04771094970608325, + "P1_FT03Z": 0.04231150648354076, + "P1_LCV01D": 0.06386369252579771, + "P1_LCV01Z": 0.0746059373701079, + "P1_LIT01": 0.21503863132655632, + "P1_PCV01D": 0.1840930329312802, + "P1_PCV01Z": 0.03163664497083818, + "P1_PCV02Z": 0.2645384172773999, + "P1_PIT01": 0.17211823821741756, + "P1_PIT02": 0.4700706769023518, + "P1_TIT01": 0.05020562808528184, + "P1_TIT02": 0.08883356398389863, + "P2_24Vdc": 0.24364840758683906, + "P2_CO_rpm": 0.18415807867084344, + "P2_HILout": 0.18479195409451332, + "P2_MSD": 0.0, + "P2_SIT01": 0.1341062931156033, + "P2_SIT02": 0.2389362509671653, + "P2_VT01": 0.005413584986390774, + "P2_VXT02": 0.029073305836900154, + "P2_VXT03": 0.014412923697366797, + "P2_VYT02": 0.016214909362405883, + "P2_VYT03": 0.04292763614604467, + "P3_FIT01": 0.7072640319988834, + "P3_LCP01D": 0.2919455744239098, + "P3_LCV01D": 0.06485406714418551, + "P3_LIT01": 0.04579782147420475, + "P3_PIT01": 0.11610722490225522, + "P4_HT_FD": 0.03417940910042122, + "P4_HT_LD": 0.19734010596285834, + "P4_HT_PO": 0.06628087492321921, + "P4_LD": 0.038818469486833984, + "P4_ST_FD": 0.3476627915764912, + "P4_ST_GOV": 0.048032291479905176, + "P4_ST_LD": 0.067094410217161, + "P4_ST_PO": 0.04533751489177951, + "P4_ST_PT01": 0.18990334783019858, + "P4_ST_TT01": 0.11644123434474485 + }, + "discrete_jsd": { + "P1_FCV01D": 0.07118877277894892, + "P1_FCV01Z": 0.48958287128887445, + "P1_FCV02D": 0.010598967483395717, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + }, + "avg_ks": 0.35632916666666664, + "avg_jsd": 0.021975792751969963, + "avg_lag1_diff": 0.13777518542478578 +} \ No newline at end of file diff --git a/example/results/ablations/runs/no_type_routing__seed1337/eval_post.json b/example/results/ablations/runs/no_type_routing__seed1337/eval_post.json new file mode 100644 index 0000000..a35421e --- /dev/null +++ b/example/results/ablations/runs/no_type_routing__seed1337/eval_post.json @@ -0,0 +1,971 @@ +{ + "rows": 384, + "continuous_summary": { + "P1_B2004": { + "mean": 0.09193078124999997, + "std": 0.019815537746997347 + }, + "P1_B2016": { + "mean": 1.3915223437499995, + "std": 0.1393654531785485 + }, + "P1_B3004": { + "mean": 399.6991551822917, + "std": 12.518914842422626 + }, + "P1_B3005": { + "mean": 1051.685153828123, + "std": 63.59243548494965 + }, + "P1_B4002": { + "mean": 32.006918411458315, + "std": 0.5813814817344444 + }, + "P1_B4005": { + "mean": 47.9263465885417, + "std": 49.790997850542055 + }, + "P1_B400B": { + "mean": 1743.4152112500008, + "std": 1229.6701768640178 + }, + "P1_B4022": { + "mean": 36.06212914062498, + "std": 0.6831827635704265 + }, + "P1_FCV02Z": { + "mean": 18.558006718750008, + "std": 32.538737586534346 + }, + "P1_FCV03D": { + "mean": 58.570887838541644, + "std": 6.6406258388712 + }, + "P1_FCV03Z": { + "mean": 59.17428062499997, + "std": 6.677133912549545 + }, + "P1_FT01": { + "mean": 194.80419330729157, + "std": 32.02466301288328 + }, + "P1_FT01Z": { + "mean": 877.2179718489588, + "std": 84.5261318996543 + }, + "P1_FT02": { + "mean": 1128.853577473959, + "std": 894.9150736225375 + }, + "P1_FT02Z": { + "mean": 1743.4468020052082, + "std": 1231.48016631041 + }, + "P1_FT03": { + "mean": 282.4489581510417, + "std": 33.658513522739625 + }, + "P1_FT03Z": { + "mean": 1060.5806286718764, + "std": 61.63984648937954 + }, + "P1_LCV01D": { + "mean": 12.16772833333333, + "std": 3.046298692652483 + }, + "P1_LCV01Z": { + "mean": 11.847819270833337, + "std": 3.1165923387339785 + }, + "P1_LIT01": { + "mean": 403.9420300520833, + "std": 15.781781361473751 + }, + "P1_PCV01D": { + "mean": 67.44987505208331, + "std": 23.854386093147486 + }, + "P1_PCV01Z": { + "mean": 68.38738161458328, + "std": 23.947728580011706 + }, + "P1_PCV02Z": { + "mean": 11.781131171874993, + "std": 0.010866175171277234 + }, + "P1_PIT01": { + "mean": 1.3746076302083339, + "std": 0.05960672856037052 + }, + "P1_PIT02": { + "mean": 0.33484200520833357, + "std": 0.1948743524939526 + }, + "P1_TIT01": { + "mean": 35.63990026041668, + "std": 0.5343657077642556 + }, + "P1_TIT02": { + "mean": 36.52362546874998, + "std": 1.230905429705141 + }, + "P2_24Vdc": { + "mean": 28.02859052083333, + "std": 0.003421123788857911 + }, + "P2_CO_rpm": { + "mean": 54102.357055650995, + "std": 23.98807742907888 + }, + "P2_HILout": { + "mean": 711.4852836458331, + "std": 9.41984148144893 + }, + "P2_MSD": { + "mean": 763.19324, + "std": 0.0 + }, + "P2_SIT01": { + "mean": 783.4609375000005, + "std": 5.351219099895989 + }, + "P2_SIT02": { + "mean": 783.9247166145833, + "std": 5.839942048250984 + }, + "P2_VT01": { + "mean": 11.923782994791672, + "std": 0.06898982316928673 + }, + "P2_VXT02": { + "mean": -3.299330755208332, + "std": 0.47730137668575 + }, + "P2_VXT03": { + "mean": -1.3056674739583336, + "std": 0.35739736340274214 + }, + "P2_VYT02": { + "mean": 3.9808523437499996, + "std": 0.5057055033410006 + }, + "P2_VYT03": { + "mean": 6.233328177083334, + "std": 0.38944681565165135 + }, + "P3_FIT01": { + "mean": 426.19300627604184, + "std": 1326.939025355576 + }, + "P3_LCP01D": { + "mean": 4590.694557578123, + "std": 5143.367197123021 + }, + "P3_LCV01D": { + "mean": 7119.145154817703, + "std": 6880.3312187991705 + }, + "P3_LIT01": { + "mean": 13352.39418132812, + "std": 4371.087325465592 + }, + "P3_PIT01": { + "mean": -18.901041666666664, + "std": 1.4544786180892033 + }, + "P4_HT_FD": { + "mean": -0.003472005208333333, + "std": 0.012897029770787882 + }, + "P4_HT_LD": { + "mean": 23.069752760416677, + "std": 33.75028540477662 + }, + "P4_HT_PO": { + "mean": 33.93330171874999, + "std": 31.615734310851927 + }, + "P4_LD": { + "mean": 366.1256761718753, + "std": 61.02547609796988 + }, + "P4_ST_FD": { + "mean": -7.067708333333341e-05, + "std": 0.004046225660922525 + }, + "P4_ST_GOV": { + "mean": 17918.27675395833, + "std": 1895.9205860154618 + }, + "P4_ST_LD": { + "mean": 332.0906425260417, + "std": 38.58216792421147 + }, + "P4_ST_PO": { + "mean": 332.3652670833332, + "std": 35.25397160583083 + }, + "P4_ST_PT01": { + "mean": 10025.982460026047, + "std": 29.990919766020852 + }, + "P4_ST_TT01": { + "mean": 27606.179697682306, + "std": 25.153659557189588 + } + }, + "continuous_error": { + "P1_B2004": { + "mean_abs_err": 0.0016368708409757043, + "std_abs_err": 0.0018171006649049422 + }, + "P1_B2016": { + "mean_abs_err": 0.00025601908523253947, + "std_abs_err": 0.0035918907019854285 + }, + "P1_B3004": { + "mean_abs_err": 3.686908642234471, + "std_abs_err": 2.899234745619184 + }, + "P1_B3005": { + "mean_abs_err": 9.363632700506741, + "std_abs_err": 2.4091189739493686 + }, + "P1_B4002": { + "mean_abs_err": 0.3100079084215679, + "std_abs_err": 0.0027079825353665266 + }, + "P1_B4005": { + "mean_abs_err": 12.598760427656224, + "std_abs_err": 7.609700402832367 + }, + "P1_B400B": { + "mean_abs_err": 35.060658927967324, + "std_abs_err": 36.3179039449451 + }, + "P1_B4022": { + "mean_abs_err": 0.0009544932224585523, + "std_abs_err": 0.011731908735608143 + }, + "P1_FCV02Z": { + "mean_abs_err": 6.392928181722269, + "std_abs_err": 7.899747852723834 + }, + "P1_FCV03D": { + "mean_abs_err": 0.003024387694814834, + "std_abs_err": 0.041130120162445216 + }, + "P1_FCV03Z": { + "mean_abs_err": 0.12475855518986378, + "std_abs_err": 0.03767005178205718 + }, + "P1_FT01": { + "mean_abs_err": 0.08053895101130593, + "std_abs_err": 4.676681174238954 + }, + "P1_FT01Z": { + "mean_abs_err": 1.8317096551937766, + "std_abs_err": 8.760428936336055 + }, + "P1_FT02": { + "mean_abs_err": 3.9645757606672305, + "std_abs_err": 5.718909771124913 + }, + "P1_FT02Z": { + "mean_abs_err": 35.031644603440554, + "std_abs_err": 38.13481053926034 + }, + "P1_FT03": { + "mean_abs_err": 0.19001212518719512, + "std_abs_err": 0.18430041144412002 + }, + "P1_FT03Z": { + "mean_abs_err": 0.4614096766451894, + "std_abs_err": 0.5143576257622158 + }, + "P1_LCV01D": { + "mean_abs_err": 0.0133956715581931, + "std_abs_err": 0.1316837396223618 + }, + "P1_LCV01Z": { + "mean_abs_err": 0.11825475850458034, + "std_abs_err": 0.15312284138866694 + }, + "P1_LIT01": { + "mean_abs_err": 0.17284211092299984, + "std_abs_err": 0.14988331309977987 + }, + "P1_PCV01D": { + "mean_abs_err": 0.015698683608064812, + "std_abs_err": 0.08461939224191184 + }, + "P1_PCV01Z": { + "mean_abs_err": 0.020770182319310493, + "std_abs_err": 0.08312876934806113 + }, + "P1_PCV02Z": { + "mean_abs_err": 0.06162177472993413, + "std_abs_err": 0.08597829029337013 + }, + "P1_PIT01": { + "mean_abs_err": 0.002873590545058091, + "std_abs_err": 0.0018258308095115833 + }, + "P1_PIT02": { + "mean_abs_err": 0.15231508513318315, + "std_abs_err": 0.30383728556092726 + }, + "P1_TIT01": { + "mean_abs_err": 0.04130798201653363, + "std_abs_err": 0.04891491942169335 + }, + "P1_TIT02": { + "mean_abs_err": 0.0013532095865542715, + "std_abs_err": 0.009146703680877888 + }, + "P2_24Vdc": { + "mean_abs_err": 0.00011918179433934029, + "std_abs_err": 7.865767445099613e-05 + }, + "P2_CO_rpm": { + "mean_abs_err": 0.3531422316809767, + "std_abs_err": 1.5323456542905163 + }, + "P2_HILout": { + "mean_abs_err": 0.022474375736919683, + "std_abs_err": 0.9651156041927678 + }, + "P2_MSD": { + "mean_abs_err": 0.0, + "std_abs_err": 1.0 + }, + "P2_SIT01": { + "mean_abs_err": 0.44297991344251386, + "std_abs_err": 0.15230373945176368 + }, + "P2_SIT02": { + "mean_abs_err": 0.02097873203706513, + "std_abs_err": 0.6493622914380177 + }, + "P2_VT01": { + "mean_abs_err": 3.265121846141028e-05, + "std_abs_err": 0.0004761410972846658 + }, + "P2_VXT02": { + "mean_abs_err": 0.0008477171816640983, + "std_abs_err": 0.006268450414529725 + }, + "P2_VXT03": { + "mean_abs_err": 0.0012258127063442537, + "std_abs_err": 0.007769984239088479 + }, + "P2_VYT02": { + "mean_abs_err": 0.0006165062429372981, + "std_abs_err": 0.00579505034821276 + }, + "P2_VYT03": { + "mean_abs_err": 0.0010769357145505865, + "std_abs_err": 0.009643853680559755 + }, + "P3_FIT01": { + "mean_abs_err": 724.7071698301413, + "std_abs_err": 452.7856724346989 + }, + "P3_LCP01D": { + "mean_abs_err": 5.9674093375488155, + "std_abs_err": 13.367199407141925 + }, + "P3_LCV01D": { + "mean_abs_err": 14.73156216353891, + "std_abs_err": 40.403322983349426 + }, + "P3_LIT01": { + "mean_abs_err": 2.4503113658665825, + "std_abs_err": 16.912688537655413 + }, + "P3_PIT01": { + "mean_abs_err": 668.2589446899943, + "std_abs_err": 1149.467452089106 + }, + "P4_HT_FD": { + "mean_abs_err": 0.003368817262981593, + "std_abs_err": 0.010865011456383782 + }, + "P4_HT_LD": { + "mean_abs_err": 10.925164319268937, + "std_abs_err": 0.3264918947783144 + }, + "P4_HT_PO": { + "mean_abs_err": 0.05050607600691848, + "std_abs_err": 0.17486309353045115 + }, + "P4_LD": { + "mean_abs_err": 0.0030776223190400742, + "std_abs_err": 0.4096697285268718 + }, + "P4_ST_FD": { + "mean_abs_err": 4.1563853450184596e-05, + "std_abs_err": 0.0024040494029324956 + }, + "P4_ST_GOV": { + "mean_abs_err": 5.315492710804392, + "std_abs_err": 43.77737002768845 + }, + "P4_ST_LD": { + "mean_abs_err": 0.08774535622399071, + "std_abs_err": 0.7289986325939211 + }, + "P4_ST_PO": { + "mean_abs_err": 0.09629575967443316, + "std_abs_err": 0.8030281034225268 + }, + "P4_ST_PT01": { + "mean_abs_err": 24.59269679303543, + "std_abs_err": 12.206149740701722 + }, + "P4_ST_TT01": { + "mean_abs_err": 2.852993997224985, + "std_abs_err": 1.2363377526212602 + } + }, + "discrete_invalid_counts": { + "P1_FCV01D": 0, + "P1_FCV01Z": 0, + "P1_FCV02D": 0, + "P1_PCV02D": 0, + "P1_PP01AD": 0, + "P1_PP01AR": 0, + "P1_PP01BD": 0, + "P1_PP01BR": 0, + "P1_PP02D": 0, + "P1_PP02R": 0, + "P1_STSP": 0, + "P2_ASD": 0, + "P2_AutoGO": 0, + "P2_Emerg": 0, + "P2_ManualGO": 0, + "P2_OnOff": 0, + "P2_RTR": 0, + "P2_TripEx": 0, + "P2_VTR01": 0, + "P2_VTR02": 0, + "P2_VTR03": 0, + "P2_VTR04": 0, + "P3_LH": 0, + "P3_LL": 0, + "P4_HT_PS": 0, + "P4_ST_PS": 0 + }, + "missing_generated_columns": {}, + "continuous_ks": { + "P1_B2004": 0.701225, + "P1_B2016": 0.04325000000000001, + "P1_B3004": 0.6640625, + "P1_B3005": 0.5657583333333334, + "P1_B4002": 0.81055, + "P1_B4005": 0.42952916666666663, + "P1_B400B": 0.45285833333333336, + "P1_B4022": 0.6185416666666667, + "P1_FCV02Z": 0.7563708333333333, + "P1_FCV03D": 0.4882875, + "P1_FCV03Z": 0.4923916666666667, + "P1_FT01": 0.4750083333333333, + "P1_FT01Z": 0.4750083333333333, + "P1_FT02": 0.4530583333333334, + "P1_FT02Z": 0.4536416666666666, + "P1_FT03": 0.5244541666666667, + "P1_FT03Z": 0.5245541666666667, + "P1_LCV01D": 0.4472333333333333, + "P1_LCV01Z": 0.4523125, + "P1_LIT01": 0.49126250000000005, + "P1_PCV01D": 0.5577125, + "P1_PCV01Z": 0.5588041666666668, + "P1_PCV02Z": 1.0, + "P1_PIT01": 0.1511041666666666, + "P1_PIT02": 0.6707083333333334, + "P1_TIT01": 0.17708749999999995, + "P1_TIT02": 0.10090833333333338, + "P2_24Vdc": 0.11985833333333334, + "P2_CO_rpm": 0.09294999999999998, + "P2_HILout": 0.0826, + "P2_MSD": 0.0, + "P2_SIT01": 0.4645208333333333, + "P2_SIT02": 0.5016666666666667, + "P2_VT01": 0.12326250000000005, + "P2_VXT02": 0.2816291666666666, + "P2_VXT03": 0.43657500000000005, + "P2_VYT02": 0.25022083333333334, + "P2_VYT03": 0.26955416666666665, + "P3_FIT01": 0.38516249999999996, + "P3_LCP01D": 0.01715833333333333, + "P3_LCV01D": 0.172725, + "P3_LIT01": 0.09314166666666668, + "P3_PIT01": 0.43189999999999995, + "P4_HT_FD": 0.2294, + "P4_HT_LD": 0.4056375, + "P4_HT_PO": 0.1821708333333333, + "P4_LD": 0.031329166666666686, + "P4_ST_FD": 0.03490000000000004, + "P4_ST_GOV": 0.03517083333333337, + "P4_ST_LD": 0.032299999999999995, + "P4_ST_PO": 0.03607499999999997, + "P4_ST_PT01": 0.4944916666666666, + "P4_ST_TT01": 0.1453625 + }, + "continuous_quantile_diff": { + "P1_B2004": { + "q05_diff": 0.00022999999999999757, + "q25_diff": 0.0028399999999999953, + "q50_diff": 0.0015000000000000013, + "q75_diff": 0.0003399999999999931, + "q95_diff": 0.00033000000000001084 + }, + "P1_B2016": { + "q05_diff": 0.01200000000000001, + "q25_diff": 0.010310000000000041, + "q50_diff": 0.005179999999999962, + "q75_diff": 0.014189999999999925, + "q95_diff": 0.030819999999999848 + }, + "P1_B3004": { + "q05_diff": 17.269769999999994, + "q25_diff": 1.658420000000035, + "q50_diff": 6.61248999999998, + "q75_diff": 8.571069999999963, + "q95_diff": 18.877499999999998 + }, + "P1_B3005": { + "q05_diff": 92.21636999999998, + "q25_diff": 3.1389700000000857, + "q50_diff": 89.51202, + "q75_diff": 103.22381000000007, + "q95_diff": 4.042119999999841 + }, + "P1_B4002": { + "q05_diff": 0.5865699999999983, + "q25_diff": 2.242070000000002, + "q50_diff": 1.6555000000000035, + "q75_diff": 1.118700000000004, + "q95_diff": 0.19520000000000692 + }, + "P1_B4005": { + "q05_diff": 0.0, + "q25_diff": 100.0, + "q50_diff": 100.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P1_B400B": { + "q05_diff": 1.986220000000003, + "q25_diff": 2526.96808, + "q50_diff": 191.70996999999988, + "q75_diff": 9.545890000000327, + "q95_diff": 7.019040000000132 + }, + "P1_B4022": { + "q05_diff": 0.36859000000000464, + "q25_diff": 1.335410000000003, + "q50_diff": 1.4134700000000038, + "q75_diff": 1.1175700000000006, + "q95_diff": 0.4899300000000011 + }, + "P1_FCV02Z": { + "q05_diff": 0.007630000000000026, + "q25_diff": 0.015260000000000051, + "q50_diff": 0.07630000000000003, + "q75_diff": 43.01453, + "q95_diff": 0.022880000000000678 + }, + "P1_FCV03D": { + "q05_diff": 2.89734, + "q25_diff": 0.09548000000000201, + "q50_diff": 8.54806, + "q75_diff": 11.147739999999999, + "q95_diff": 0.6383199999999931 + }, + "P1_FCV03Z": { + "q05_diff": 3.2806399999999982, + "q25_diff": 0.10681000000000296, + "q50_diff": 8.96454, + "q75_diff": 11.337280000000007, + "q95_diff": 0.4730199999999911 + }, + "P1_FT01": { + "q05_diff": 7.82011, + "q25_diff": 8.588690000000014, + "q50_diff": 32.04345999999998, + "q75_diff": 37.76549, + "q95_diff": 3.0517600000000016 + }, + "P1_FT01Z": { + "q05_diff": 39.17327999999998, + "q25_diff": 19.58227999999997, + "q50_diff": 74.36358000000007, + "q75_diff": 86.10528, + "q95_diff": 6.958009999999945 + }, + "P1_FT02": { + "q05_diff": 0.5722499999999995, + "q25_diff": 1935.76794, + "q50_diff": 268.60411999999997, + "q75_diff": 12.970209999999952, + "q95_diff": 9.536740000000009 + }, + "P1_FT02Z": { + "q05_diff": 8.937850000000001, + "q25_diff": 2770.27588, + "q50_diff": 189.2133799999997, + "q75_diff": 9.545890000000327, + "q95_diff": 7.019040000000132 + }, + "P1_FT03": { + "q05_diff": 36.04084, + "q25_diff": 0.953630000000004, + "q50_diff": 51.52633000000003, + "q75_diff": 59.761000000000024, + "q95_diff": 1.555970000000002 + }, + "P1_FT03Z": { + "q05_diff": 81.72820999999999, + "q25_diff": 2.672670000000039, + "q50_diff": 87.85290000000009, + "q75_diff": 100.88019000000008, + "q95_diff": 2.282600000000002 + }, + "P1_LCV01D": { + "q05_diff": 0.1012599999999999, + "q25_diff": 1.2061399999999995, + "q50_diff": 4.635400000000001, + "q75_diff": 3.3114999999999988, + "q95_diff": 0.61768 + }, + "P1_LCV01Z": { + "q05_diff": 0.2136199999999997, + "q25_diff": 1.1901799999999998, + "q50_diff": 4.76075, + "q75_diff": 3.4561199999999985, + "q95_diff": 0.7136000000000013 + }, + "P1_LIT01": { + "q05_diff": 14.721709999999973, + "q25_diff": 1.0259100000000103, + "q50_diff": 6.23626999999999, + "q75_diff": 8.178740000000005, + "q95_diff": 34.780489999999986 + }, + "P1_PCV01D": { + "q05_diff": 1.0988299999999995, + "q25_diff": 3.711590000000001, + "q50_diff": 22.252399999999994, + "q75_diff": 44.83375, + "q95_diff": 46.32521 + }, + "P1_PCV01Z": { + "q05_diff": 1.0054599999999994, + "q25_diff": 3.814699999999995, + "q50_diff": 22.764490000000002, + "q75_diff": 45.46356000000001, + "q95_diff": 45.1455 + }, + "P1_PCV02Z": { + "q05_diff": 0.2441399999999998, + "q25_diff": 0.2441399999999998, + "q50_diff": 0.2288800000000002, + "q75_diff": 0.2288800000000002, + "q95_diff": 0.2288800000000002 + }, + "P1_PIT01": { + "q05_diff": 0.03492000000000006, + "q25_diff": 0.017550000000000177, + "q50_diff": 0.0045800000000000285, + "q75_diff": 0.022119999999999918, + "q95_diff": 0.019830000000000014 + }, + "P1_PIT02": { + "q05_diff": 0.03051999999999999, + "q25_diff": 0.0015199999999999658, + "q50_diff": 0.0022899999999999587, + "q75_diff": 0.003049999999999997, + "q95_diff": 0.46234 + }, + "P1_TIT01": { + "q05_diff": 0.07628000000000412, + "q25_diff": 0.07629999999999626, + "q50_diff": 0.0915600000000012, + "q75_diff": 0.2593999999999994, + "q95_diff": 0.27465999999999724 + }, + "P1_TIT02": { + "q05_diff": 0.01525999999999783, + "q25_diff": 0.0614400000000046, + "q50_diff": 0.09154999999999802, + "q75_diff": 0.29144999999999754, + "q95_diff": 0.43856999999999857 + }, + "P2_24Vdc": { + "q05_diff": 3.999999999848569e-05, + "q25_diff": 0.0008400000000001739, + "q50_diff": 0.0008799999999986596, + "q75_diff": 0.0008799999999986596, + "q95_diff": 0.000920000000000698 + }, + "P2_CO_rpm": { + "q05_diff": 10.0, + "q25_diff": 5.0, + "q50_diff": 3.0, + "q75_diff": 2.0, + "q95_diff": 1.0 + }, + "P2_HILout": { + "q05_diff": 0.5802699999999277, + "q25_diff": 2.0135499999998956, + "q50_diff": 0.08672999999998865, + "q75_diff": 1.5440100000000712, + "q95_diff": 1.1977000000000544 + }, + "P2_MSD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + }, + "P2_SIT01": { + "q05_diff": 3.0, + "q25_diff": 3.0, + "q50_diff": 5.0, + "q75_diff": 6.0, + "q95_diff": 6.0 + }, + "P2_SIT02": { + "q05_diff": 2.660160000000019, + "q25_diff": 3.5689700000000357, + "q50_diff": 5.618170000000077, + "q75_diff": 6.357729999999947, + "q95_diff": 6.8461300000000165 + }, + "P2_VT01": { + "q05_diff": 0.00835000000000008, + "q25_diff": 0.015249999999999986, + "q50_diff": 0.014160000000000394, + "q75_diff": 0.01328999999999958, + "q95_diff": 0.018950000000000244 + }, + "P2_VXT02": { + "q05_diff": 0.13046000000000024, + "q25_diff": 0.2993999999999999, + "q50_diff": 0.28474999999999984, + "q75_diff": 0.26602000000000015, + "q95_diff": 0.4596100000000001 + }, + "P2_VXT03": { + "q05_diff": 0.10050999999999988, + "q25_diff": 0.2691699999999999, + "q50_diff": 0.29742, + "q75_diff": 0.32671000000000006, + "q95_diff": 0.61104 + }, + "P2_VYT02": { + "q05_diff": 0.09780999999999995, + "q25_diff": 0.26202000000000014, + "q50_diff": 0.2266300000000001, + "q75_diff": 0.23521999999999998, + "q95_diff": 0.3870300000000002 + }, + "P2_VYT03": { + "q05_diff": 0.05653000000000041, + "q25_diff": 0.18581000000000003, + "q50_diff": 0.17598999999999965, + "q75_diff": 0.18259999999999987, + "q95_diff": 0.3547899999999995 + }, + "P3_FIT01": { + "q05_diff": 1.0, + "q25_diff": 0.0, + "q50_diff": 72.0, + "q75_diff": 2582.0, + "q95_diff": 331.0 + }, + "P3_LCP01D": { + "q05_diff": 0.0, + "q25_diff": 16.0, + "q50_diff": 102.91283999999996, + "q75_diff": 120.49706999999944, + "q95_diff": 9.175779999999577 + }, + "P3_LCV01D": { + "q05_diff": 0.0, + "q25_diff": 256.0, + "q50_diff": 3238.2841799999997, + "q75_diff": 168.74316, + "q95_diff": 30.367190000000846 + }, + "P3_LIT01": { + "q05_diff": 1135.8911099999996, + "q25_diff": 26.31151999999929, + "q50_diff": 649.4033199999994, + "q75_diff": 1140.2734400000008, + "q95_diff": 2.3339799999994284 + }, + "P3_PIT01": { + "q05_diff": 2.0, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 928.0, + "q95_diff": 3524.0 + }, + "P4_HT_FD": { + "q05_diff": 0.048029999999999996, + "q25_diff": 0.00014, + "q50_diff": 7.000000000000002e-05, + "q75_diff": 7e-05, + "q95_diff": 0.00021999999999999993 + }, + "P4_HT_LD": { + "q05_diff": 0.0, + "q25_diff": 0.0, + "q50_diff": 54.74537, + "q75_diff": 0.5989600000000053, + "q95_diff": 0.39124999999999943 + }, + "P4_HT_PO": { + "q05_diff": 0.0, + "q25_diff": 1.33825, + "q50_diff": 14.777420000000003, + "q75_diff": 0.785899999999998, + "q95_diff": 0.09572000000000003 + }, + "P4_LD": { + "q05_diff": 0.5577400000000239, + "q25_diff": 2.9282499999999914, + "q50_diff": 3.4519900000000234, + "q75_diff": 0.16329999999999245, + "q95_diff": 4.151920000000018 + }, + "P4_ST_FD": { + "q05_diff": 2.9999999999999645e-05, + "q25_diff": 0.0, + "q50_diff": 0.0, + "q75_diff": 0.00014000000000000004, + "q95_diff": 8.000000000000021e-05 + }, + "P4_ST_GOV": { + "q05_diff": 119.87890999999945, + "q25_diff": 38.54102000000057, + "q50_diff": 31.49022999999943, + "q75_diff": 109.1992199999986, + "q95_diff": 417.9902299999994 + }, + "P4_ST_LD": { + "q05_diff": 0.7481400000000349, + "q25_diff": 0.41091999999997597, + "q50_diff": 0.47870000000000346, + "q75_diff": 1.9949300000000108, + "q95_diff": 7.954279999999983 + }, + "P4_ST_PO": { + "q05_diff": 2.229069999999979, + "q25_diff": 0.732910000000004, + "q50_diff": 0.5704400000000192, + "q75_diff": 2.1875, + "q95_diff": 7.791440000000023 + }, + "P4_ST_PT01": { + "q05_diff": 23.0, + "q25_diff": 31.0, + "q50_diff": 54.0, + "q75_diff": 0.0, + "q95_diff": 18.0 + }, + "P4_ST_TT01": { + "q05_diff": 5.0, + "q25_diff": 3.0, + "q50_diff": 4.0, + "q75_diff": 0.0, + "q95_diff": 0.0 + } + }, + "continuous_lag1_diff": { + "P1_B2004": 0.0842454873511631, + "P1_B2016": 0.031142256345468855, + "P1_B3004": 0.09059121934513292, + "P1_B3005": 0.10933425266689745, + "P1_B4002": 0.1381657367610738, + "P1_B4005": 0.044394423231015945, + "P1_B400B": 0.2302252103627388, + "P1_B4022": 0.02288387249075763, + "P1_FCV02Z": 0.1551545855219576, + "P1_FCV03D": 0.020915544759740334, + "P1_FCV03Z": 0.03143040195289626, + "P1_FT01": 0.2608931050318283, + "P1_FT01Z": 0.22837148157422993, + "P1_FT02": 0.46676963996583987, + "P1_FT02Z": 0.21179817615122698, + "P1_FT03": 0.04771094970608325, + "P1_FT03Z": 0.04231150648354076, + "P1_LCV01D": 0.06386369252579771, + "P1_LCV01Z": 0.0746059373701079, + "P1_LIT01": 0.21503863132655632, + "P1_PCV01D": 0.1840930329312802, + "P1_PCV01Z": 0.03163664497083818, + "P1_PCV02Z": 0.2645384172773999, + "P1_PIT01": 0.17211823821741756, + "P1_PIT02": 0.4700706769023518, + "P1_TIT01": 0.05020562808528184, + "P1_TIT02": 0.08883356398389863, + "P2_24Vdc": 0.24364840758683906, + "P2_CO_rpm": 0.18415807867084344, + "P2_HILout": 0.18479195409451332, + "P2_MSD": 0.0, + "P2_SIT01": 0.1341062931156033, + "P2_SIT02": 0.2389362509671653, + "P2_VT01": 0.005413584986390774, + "P2_VXT02": 0.029073305836900154, + "P2_VXT03": 0.014412923697366797, + "P2_VYT02": 0.016214909362405883, + "P2_VYT03": 0.04292763614604467, + "P3_FIT01": 0.7072640319988834, + "P3_LCP01D": 0.2919455744239098, + "P3_LCV01D": 0.06485406714418551, + "P3_LIT01": 0.04579782147420475, + "P3_PIT01": 0.11610722490225522, + "P4_HT_FD": 0.03417940910042122, + "P4_HT_LD": 0.19734010596285834, + "P4_HT_PO": 0.06628087492321921, + "P4_LD": 0.038818469486833984, + "P4_ST_FD": 0.3476627915764912, + "P4_ST_GOV": 0.048032291479905176, + "P4_ST_LD": 0.067094410217161, + "P4_ST_PO": 0.04533751489177951, + "P4_ST_PT01": 0.18990334783019858, + "P4_ST_TT01": 0.11644123434474485 + }, + "discrete_jsd": { + "P1_FCV01D": 0.07118877277894892, + "P1_FCV01Z": 0.48958287128887445, + "P1_FCV02D": 0.010598967483395717, + "P1_PCV02D": 0.0, + "P1_PP01AD": 0.0, + "P1_PP01AR": 0.0, + "P1_PP01BD": 0.0, + "P1_PP01BR": 0.0, + "P1_PP02D": 0.0, + "P1_PP02R": 0.0, + "P1_STSP": 0.0, + "P2_ASD": 0.0, + "P2_AutoGO": 0.0, + "P2_Emerg": 0.0, + "P2_ManualGO": 0.0, + "P2_OnOff": 0.0, + "P2_RTR": 0.0, + "P2_TripEx": 0.0, + "P2_VTR01": 0.0, + "P2_VTR02": 0.0, + "P2_VTR03": 0.0, + "P2_VTR04": 0.0, + "P3_LH": 0.0, + "P3_LL": 0.0, + "P4_HT_PS": 0.0, + "P4_ST_PS": 0.0 + } +} \ No newline at end of file diff --git a/example/results/comprehensive_eval.json b/example/results/comprehensive_eval.json new file mode 100644 index 0000000..78973b3 --- /dev/null +++ b/example/results/comprehensive_eval.json @@ -0,0 +1,537 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/generated.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": 0.40248121069182385, + "avg_jsd": 0.01659609083832257, + "avg_lag1_diff": 0.2859088424512934 + }, + "two_sample": { + "continuous_mmd_rbf": 0.6498599052429199, + "continuous_mmd_gamma": 1.6837304415964555e-05, + "histogram_mmd_rbf": 0.649860143661499, + "histogram_mmd_gamma": 1.6836341031662582e-05, + "discriminative_accuracy": 1.0, + "discriminative_balanced_accuracy": 0.5, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 162.85183715820312, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 2.951477773848524, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.213444784283638, + "corr_frobenius": 16.677379608154297, + "lag1_corr_mean_abs_diff": 0.21316678822040558, + "lag1_corr_frobenius": 16.745145797729492, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.29572203755378723, + "corr_frobenius": 10.480631828308105, + "lag1_corr_mean_abs_diff": 0.31045666337013245, + "lag1_corr_frobenius": 10.77681827545166 + }, + "P2": { + "corr_mean_abs_diff": 0.11664481461048126, + "corr_frobenius": 2.4467108249664307, + "lag1_corr_mean_abs_diff": 0.15354882180690765, + "lag1_corr_frobenius": 2.759093999862671 + }, + "P3": { + "corr_mean_abs_diff": 0.4343887269496918, + "corr_frobenius": 3.0039377212524414, + "lag1_corr_mean_abs_diff": 0.49758461117744446, + "lag1_corr_frobenius": 3.1910462379455566 + }, + "P4": { + "corr_mean_abs_diff": 0.1737484186887741, + "corr_frobenius": 2.689737319946289, + "lag1_corr_mean_abs_diff": 0.19204501807689667, + "lag1_corr_frobenius": 2.8761579990386963 + } + } + }, + "frequency": { + "avg_psd_l1": 0.019466130062937737, + "avg_psd_cosine": 0.3419710143641555, + "avg_low_high_ratio_abs_diff": 9433975.2056802 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 15.0, + "mean_dwell": 24.0, + "median_dwell": 2.5, + "mean_step": 68.42660395304362, + "median_step": 1.9043693542480469 + }, + "P1_B2004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B3004": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.01115664489586733, + "median_step": 0.004810333251953125 + }, + "P1_B3005": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22998.14285714286, + "mean_abs_diff_mean_dwell": 14201.625685203559, + "mean_abs_diff_median_dwell": 14207.642857142857, + "mean_abs_diff_mean_step": 33.693117622162895, + "mean_abs_diff_median_step": 0.831573486328125 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 1.0, + "change_rate": 0.0, + "step_median": NaN + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.411041259765625, + "mean_abs_diff_change_rate": 0.4608201173921447 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.0026041666666666665, + "top1_mass": 1.0, + "top3_mass": 1.0, + "median_dwell": 384.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8671875, + "top1_mass": 0.044270833333333336, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9192708333333334, + "top1_mass": 0.08333333333333333, + "top3_mass": 0.08854166666666667, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.03125, + "top1_mass": 0.4036458333333333, + "top3_mass": 0.6953125, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.9947916666666666, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.013020833333333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.3854166666666667, + "top1_mass": 0.1015625, + "top3_mass": 0.21614583333333334, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.9635416666666666, + "top1_mass": 0.005208333333333333, + "top3_mass": 0.015625, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.1953125, + "top1_mass": 0.21614583333333334, + "top3_mass": 0.3541666666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5198198954264323, + "mean_abs_diff_top1_mass": 0.11710739135742188, + "mean_abs_diff_top3_mass": 0.08729680379231772, + "mean_abs_diff_median_dwell": 47.75 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17409999668598175, + "q50": 0.17409999668598175, + "q95": 0.2763400077819824, + "tail_ratio": NaN + }, + "P2_SIT02": { + "q05": 774.8864135742188, + "q50": 784.5529174804688, + "q95": 791.694091796875, + "tail_ratio": 0.738754609284235 + }, + "P1_FT03": { + "q05": 209.0201873779297, + "q50": 298.9196472167969, + "q95": 316.0858154296875, + "tail_ratio": 0.190948513413303 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.5982364962498345, + "mean_abs_diff_q50": 2.26713257531325, + "mean_abs_diff_q95": 3.5022283792495728, + "mean_abs_diff_tail_ratio": 0.04664732240855185 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 381.0, + "mean_dwell": 1.0052356020942408, + "median_dwell": 1.0, + "mean_step": 17.80426273696379, + "median_step": 7.81744384765625 + }, + "P1_FT02Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 589.764816448563, + "median_step": 114.94613671302795 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89282.0, + "mean_abs_diff_mean_dwell": 0.09437569141112279, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 296.133439017248, + "mean_abs_diff_median_step": 58.11488354206085 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 33.9776496887207, + "std": 31.549665451049805, + "lag1": 0.5593345165252686 + }, + "P2_24Vdc": { + "mean": 28.028594970703125, + "std": 0.003535544965416193, + "lag1": 0.28336605429649353 + }, + "P2_HILout": { + "mean": 711.4856567382812, + "std": 9.42089557647705, + "lag1": 0.050835929811000824 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.289700190226237, + "mean_abs_diff_std": 0.47051989651905995, + "mean_abs_diff_lag1": 0.2913659450908502 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9721695780754089, + "mae": 0.7823513150215149 + }, + "real_plus_synthetic": { + "rmse": 0.543311357498169, + "mae": 0.3526590168476105 + }, + "rmse_ratio_synth_to_real": 1.741144398752548 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.8251159554730984, + "auprc": 0.5888886636758117, + "threshold": 0.013622067868709564, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6756756756756758, + "best_f1_threshold": 1.4749367237091064 + }, + "real_plus_synthetic": { + "auroc": 0.8797309833024117, + "auprc": 0.6449370959119753, + "threshold": 0.3079744577407837, + "f1": 0.2622950819672131, + "precision": 0.16, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.5546749830245972 + } + } +} \ No newline at end of file diff --git a/example/results/comprehensive_eval_post.json b/example/results/comprehensive_eval_post.json new file mode 100644 index 0000000..6922f4c --- /dev/null +++ b/example/results/comprehensive_eval_post.json @@ -0,0 +1,538 @@ +{ + "generated_path": "/root/autodl-tmp/mask-ddpm/example/results/generated_post.csv", + "reference_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/train1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/train3.csv.gz" + ], + "test_paths": [ + "/root/autodl-tmp/dataset/hai/hai-21.03/test1.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test2.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test3.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test4.csv.gz", + "/root/autodl-tmp/dataset/hai/hai-21.03/test5.csv.gz" + ], + "seq_len": 96, + "stride": 96, + "counts": { + "train_windows": 1024, + "generated_windows": 4, + "test_windows": 1024, + "test_anomalous_windows": 44, + "test_normal_windows": 980 + }, + "basic_eval": { + "avg_ks": null, + "avg_jsd": null, + "avg_lag1_diff": null + }, + "two_sample": { + "continuous_mmd_rbf": 0.21657896041870117, + "continuous_mmd_gamma": 5.317558463803058e-05, + "histogram_mmd_rbf": 0.216666579246521, + "histogram_mmd_gamma": 5.316644607437446e-05, + "discriminative_accuracy": 0.5, + "discriminative_balanced_accuracy": 0.25, + "discriminative_auroc": NaN + }, + "diversity_privacy": { + "duplicate_rate": 0.0, + "exact_match_rate_to_train": 0.0, + "nn_gen_to_train_mean": 89.41130065917969, + "nn_holdout_to_train_mean": 55.17637252807617, + "memorization_ratio": 1.6204635528311193, + "one_nn_two_sample_accuracy": 1.0 + }, + "coupling": { + "corr_mean_abs_diff": 0.19092726707458496, + "corr_frobenius": 15.937767028808594, + "lag1_corr_mean_abs_diff": 0.19889004528522491, + "lag1_corr_frobenius": 16.619050979614258, + "by_process": { + "P1": { + "corr_mean_abs_diff": 0.3421248197555542, + "corr_frobenius": 11.90279483795166, + "lag1_corr_mean_abs_diff": 0.3640459179878235, + "lag1_corr_frobenius": 12.561296463012695 + }, + "P2": { + "corr_mean_abs_diff": 0.08267470449209213, + "corr_frobenius": 1.617314338684082, + "lag1_corr_mean_abs_diff": 0.14486488699913025, + "lag1_corr_frobenius": 2.7614989280700684 + }, + "P3": { + "corr_mean_abs_diff": 0.4343887269496918, + "corr_frobenius": 3.0039377212524414, + "lag1_corr_mean_abs_diff": 0.49758461117744446, + "lag1_corr_frobenius": 3.1910462379455566 + }, + "P4": { + "corr_mean_abs_diff": 0.20006132125854492, + "corr_frobenius": 3.197087049484253, + "lag1_corr_mean_abs_diff": 0.23344576358795166, + "lag1_corr_frobenius": 3.5848729610443115 + } + } + }, + "frequency": { + "avg_psd_l1": 0.02240946888923645, + "avg_psd_cosine": 0.4469434787642281, + "avg_low_high_ratio_abs_diff": 18.3366464166149 + }, + "type_metrics": { + "type1_program": { + "features": [ + "P1_B4002", + "P2_MSD", + "P4_HT_LD", + "P1_B2004", + "P1_B3004", + "P1_B4022", + "P1_B3005" + ], + "generated": { + "P1_B4002": { + "num_changes": 308.0, + "mean_dwell": 1.2427184466019416, + "median_dwell": 1.0, + "mean_step": 0.8413416007896523, + "median_step": 0.7274894714355469 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 384.0, + "median_dwell": 384.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 347.0, + "mean_dwell": 1.103448275862069, + "median_dwell": 1.0, + "mean_step": 41.3141472878254, + "median_step": 54.116028785705566 + }, + "P1_B2004": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 0.014068839402653235, + "median_step": 0.0017450004816055298 + }, + "P1_B3004": { + "num_changes": 370.0, + "mean_dwell": 1.0350404312668464, + "median_dwell": 1.0, + "mean_step": 14.77256890374261, + "median_step": 10.804779052734375 + }, + "P1_B4022": { + "num_changes": 383.0, + "mean_dwell": 1.0, + "median_dwell": 1.0, + "mean_step": 0.8157603410765646, + "median_step": 0.5944099426269531 + }, + "P1_B3005": { + "num_changes": 362.0, + "mean_dwell": 1.0578512396694215, + "median_dwell": 1.0, + "mean_step": 64.28072175926926, + "median_step": 41.228179931640625 + } + }, + "reference": { + "P1_B4002": { + "num_changes": 97.0, + "mean_dwell": 1003.1020408163265, + "median_dwell": 1.0, + "mean_step": 0.032477270696581025, + "median_step": 0.0126495361328125 + }, + "P2_MSD": { + "num_changes": 0.0, + "mean_dwell": 98304.0, + "median_dwell": 98304.0, + "mean_step": NaN, + "median_step": NaN + }, + "P4_HT_LD": { + "num_changes": 63727.0, + "mean_dwell": 1.5425558624152649, + "median_dwell": 1.0, + "mean_step": 1.0409184914880636, + "median_step": 0.245941162109375 + }, + "P1_B2004": { + "num_changes": 145.0, + "mean_dwell": 673.3150684931506, + "median_dwell": 1.0, + "mean_step": 0.0015369655757114806, + "median_step": 0.00010000169277191162 + }, + "P1_B3004": { + "num_changes": 146.0, + "mean_dwell": 668.734693877551, + "median_dwell": 1.0, + "mean_step": 1.113193877755779, + "median_step": 0.819305419921875 + }, + "P1_B4022": { + "num_changes": 97122.0, + "mean_dwell": 1.0121598385552342, + "median_dwell": 1.0, + "mean_step": 0.011706427666086412, + "median_step": 0.00952911376953125 + }, + "P1_B3005": { + "num_changes": 148.0, + "mean_dwell": 659.7583892617449, + "median_dwell": 1.0, + "mean_step": 5.635673316749367, + "median_step": 5.34417724609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 22997.85714285714, + "mean_abs_diff_mean_dwell": 14417.281142645239, + "mean_abs_diff_median_dwell": 13988.57142857143, + "mean_abs_diff_mean_step": 19.03385039702909, + "mean_abs_diff_median_step": 16.840154950817425 + } + }, + "type2_controller": { + "features": [ + "P1_B4005" + ], + "generated": { + "P1_B4005": { + "saturation_ratio": 0.6822916666666666, + "change_rate": 0.7702349869451697, + "step_median": 68.81373023986816 + } + }, + "reference": { + "P1_B4005": { + "saturation_ratio": 0.588958740234375, + "change_rate": 0.4608201173921447, + "step_median": 0.03202030062675476 + } + }, + "aggregates": { + "mean_abs_diff_saturation_ratio": 0.09333292643229163, + "mean_abs_diff_change_rate": 0.309414869553025, + "mean_abs_diff_step_median": 68.78170993924141 + } + }, + "type3_actuator": { + "features": [ + "P1_PCV02Z", + "P1_PCV01Z", + "P1_PCV01D", + "P1_FCV02Z", + "P1_FCV03D", + "P1_FCV03Z", + "P1_LCV01D", + "P1_LCV01Z" + ], + "generated": { + "P1_PCV02Z": { + "unique_ratio": 0.015625, + "top1_mass": 0.40625, + "top3_mass": 0.7760416666666666, + "median_dwell": 1.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.8776041666666666, + "top1_mass": 0.0078125, + "top3_mass": 0.0234375, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.9166666666666666, + "top1_mass": 0.059895833333333336, + "top3_mass": 0.0703125, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.22135416666666666, + "top1_mass": 0.18229166666666666, + "top3_mass": 0.3697916666666667, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.8463541666666666, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.4817708333333333, + "top1_mass": 0.033854166666666664, + "top3_mass": 0.0859375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.7864583333333334, + "top1_mass": 0.010416666666666666, + "top3_mass": 0.026041666666666668, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.3515625, + "top1_mass": 0.036458333333333336, + "top3_mass": 0.09635416666666667, + "median_dwell": 1.0 + } + }, + "reference": { + "P1_PCV02Z": { + "unique_ratio": 5.0862630208333336e-05, + "top1_mass": 0.5122273763020834, + "top3_mass": 0.9970194498697916, + "median_dwell": 2.0 + }, + "P1_PCV01Z": { + "unique_ratio": 0.063232421875, + "top1_mass": 0.009714762369791666, + "top3_mass": 0.020904541015625, + "median_dwell": 1.0 + }, + "P1_PCV01D": { + "unique_ratio": 0.074310302734375, + "top1_mass": 0.043843587239583336, + "top3_mass": 0.044850667317708336, + "median_dwell": 1.0 + }, + "P1_FCV02Z": { + "unique_ratio": 0.058146158854166664, + "top1_mass": 0.20765177408854166, + "top3_mass": 0.3501078287760417, + "median_dwell": 1.0 + }, + "P1_FCV03D": { + "unique_ratio": 0.017527262369791668, + "top1_mass": 0.004384358723958333, + "top3_mass": 0.012776692708333334, + "median_dwell": 1.0 + }, + "P1_FCV03Z": { + "unique_ratio": 0.013102213541666666, + "top1_mass": 0.10939534505208333, + "top3_mass": 0.2132568359375, + "median_dwell": 1.0 + }, + "P1_LCV01D": { + "unique_ratio": 0.015523274739583334, + "top1_mass": 0.0033976236979166665, + "top3_mass": 0.009999593098958334, + "median_dwell": 1.0 + }, + "P1_LCV01Z": { + "unique_ratio": 0.012715657552083334, + "top1_mass": 0.047566731770833336, + "top3_mass": 0.1214599609375, + "median_dwell": 1.0 + } + }, + "aggregates": { + "mean_abs_diff_unique_ratio": 0.5303484598795573, + "mean_abs_diff_top1_mass": 0.03112411499023438, + "mean_abs_diff_top3_mass": 0.05629857381184896, + "mean_abs_diff_median_dwell": 0.125 + } + }, + "type4_pv": { + "features": [ + "P1_PIT02", + "P2_SIT02", + "P1_FT03" + ], + "generated": { + "P1_PIT02": { + "q05": 0.17409999668598175, + "q50": 0.17409999668598175, + "q95": 0.2763400077819824, + "tail_ratio": NaN + }, + "P2_SIT02": { + "q05": 774.8864135742188, + "q50": 784.5529174804688, + "q95": 791.694091796875, + "tail_ratio": 0.738754609284235 + }, + "P1_FT03": { + "q05": 209.0201873779297, + "q50": 298.9196472167969, + "q95": 316.0858154296875, + "tail_ratio": 0.190948513413303 + } + }, + "reference": { + "P1_PIT02": { + "q05": 0.20232999324798584, + "q50": 0.2763400077819824, + "q95": 1.8746999502182007, + "tail_ratio": 21.596535989085773 + }, + "P2_SIT02": { + "q05": 771.5216674804688, + "q50": 779.1593017578125, + "q95": 785.1046752929688, + "tail_ratio": 0.7784312942022615 + }, + "P1_FT03": { + "q05": 201.6184539794922, + "q50": 300.2251892089844, + "q95": 313.76690673828125, + "tail_ratio": 0.13733055351422585 + } + }, + "aggregates": { + "mean_abs_diff_q05": 3.5982364962498345, + "mean_abs_diff_q50": 2.26713257531325, + "mean_abs_diff_q95": 3.5022283792495728, + "mean_abs_diff_tail_ratio": 0.04664732240855185 + } + }, + "type5_program_proxy": { + "features": [ + "P1_FT03Z", + "P1_FT02Z" + ], + "generated": { + "P1_FT03Z": { + "num_changes": 380.0, + "mean_dwell": 1.0078740157480315, + "median_dwell": 1.0, + "mean_step": 64.74391126130756, + "median_step": 62.410614013671875 + }, + "P1_FT02Z": { + "num_changes": 378.0, + "mean_dwell": 1.0131926121372032, + "median_dwell": 1.0, + "mean_step": 1312.3134692358592, + "median_step": 1284.6439208984375 + } + }, + "reference": { + "P1_FT03Z": { + "num_changes": 83889.0, + "mean_dwell": 1.1718202407915126, + "median_dwell": 1.0, + "mean_step": 0.9289188007355237, + "median_step": 0.6378173828125 + }, + "P1_FT02Z": { + "num_changes": 95436.0, + "mean_dwell": 1.0300407598730053, + "median_dwell": 1.0, + "mean_step": 14.373282350295296, + "median_step": 5.89599609375 + } + }, + "aggregates": { + "mean_abs_diff_num_changes": 89283.5, + "mean_abs_diff_mean_dwell": 0.09039718638964156, + "mean_abs_diff_median_dwell": 0.0, + "mean_abs_diff_mean_step": 680.877589673068, + "mean_abs_diff_median_step": 670.2603607177734 + } + }, + "type6_aux": { + "features": [ + "P4_HT_PO", + "P2_24Vdc", + "P2_HILout" + ], + "generated": { + "P4_HT_PO": { + "mean": 35.254512786865234, + "std": 31.650897979736328, + "lag1": -0.0927804708480835 + }, + "P2_24Vdc": { + "mean": 28.028608322143555, + "std": 0.003297113813459873, + "lag1": 0.06643600016832352 + }, + "P2_HILout": { + "mean": 711.4987182617188, + "std": 7.856828212738037, + "lag1": -0.013758387416601181 + } + }, + "reference": { + "P4_HT_PO": { + "mean": 34.35928726196289, + "std": 31.320688247680664, + "lag1": 0.9990671873092651 + }, + "P2_24Vdc": { + "mean": 28.0284366607666, + "std": 0.0032698388677090406, + "lag1": 0.05670436844229698 + }, + "P2_HILout": { + "mean": 711.9729614257812, + "std": 8.238578796386719, + "lag1": 0.2585394084453583 + } + }, + "aggregates": { + "mean_abs_diff_mean": 0.4565467834472656, + "mean_abs_diff_std": 0.2373291968833655, + "mean_abs_diff_lag1": 0.4579590285817782 + } + } + }, + "predictive_consistency": { + "real_only": { + "rmse": 0.5583509206771851, + "mae": 0.366405189037323 + }, + "synthetic_only": { + "rmse": 0.9640899300575256, + "mae": 0.7753889560699463 + }, + "real_plus_synthetic": { + "rmse": 0.5412734746932983, + "mae": 0.3464338481426239 + }, + "rmse_ratio_synth_to_real": 1.7266738431956876 + }, + "anomaly_utility": { + "real_only": { + "auroc": 0.8881493506493506, + "auprc": 0.6527744226377132, + "threshold": 0.3682355284690857, + "f1": 0.35714285714285715, + "precision": 0.24193548387096775, + "recall": 0.6818181818181818, + "best_f1": 0.763157894736842, + "best_f1_threshold": 0.49033087491989136 + }, + "synthetic_only": { + "auroc": 0.826461038961039, + "auprc": 0.5893875360513735, + "threshold": 0.003091409569606185, + "f1": 0.08239700374531835, + "precision": 0.04296875, + "recall": 1.0, + "best_f1": 0.6756756756756758, + "best_f1_threshold": 1.4940791130065918 + }, + "real_plus_synthetic": { + "auroc": 0.8798701298701299, + "auprc": 0.6475849946221094, + "threshold": 0.28958627581596375, + "f1": 0.24334600760456274, + "precision": 0.1461187214611872, + "recall": 0.7272727272727273, + "best_f1": 0.7466666666666666, + "best_f1_threshold": 0.526667594909668 + } + } +} \ No newline at end of file