59 lines
1.9 KiB
Python
59 lines
1.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Run two configs sequentially and print metric deltas."""
|
|
|
|
import argparse
|
|
import csv
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
def run(cmd):
|
|
print("running:", " ".join(cmd))
|
|
subprocess.run(cmd, check=True)
|
|
|
|
|
|
def read_last_metrics(path: Path):
|
|
if not path.exists():
|
|
return None
|
|
rows = list(csv.DictReader(path.open()))
|
|
return rows[-1] if rows else None
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Compare metrics for two configs.")
|
|
base_dir = Path(__file__).resolve().parent
|
|
parser.add_argument("--config-a", default=str(base_dir / "config_no_temporal.json"))
|
|
parser.add_argument("--config-b", default=str(base_dir / "config_temporal_strong.json"))
|
|
parser.add_argument("--device", default="auto")
|
|
args = parser.parse_args()
|
|
|
|
history = base_dir / "results" / "metrics_history.csv"
|
|
if history.exists():
|
|
history.unlink()
|
|
|
|
run([sys.executable, str(base_dir / "run_all.py"), "--config", args.config_a, "--device", args.device])
|
|
a = read_last_metrics(history)
|
|
run([sys.executable, str(base_dir / "run_all.py"), "--config", args.config_b, "--device", args.device])
|
|
b = read_last_metrics(history)
|
|
|
|
if not a or not b:
|
|
raise SystemExit("missing metrics_history.csv entries")
|
|
|
|
def f(v):
|
|
return float(v) if v is not None else None
|
|
|
|
print("baseline_avg_ks", a["avg_ks"])
|
|
print("temporal_avg_ks", b["avg_ks"])
|
|
print("delta_avg_ks", f(b["avg_ks"]) - f(a["avg_ks"]))
|
|
print("baseline_avg_jsd", a["avg_jsd"])
|
|
print("temporal_avg_jsd", b["avg_jsd"])
|
|
print("delta_avg_jsd", f(b["avg_jsd"]) - f(a["avg_jsd"]))
|
|
print("baseline_avg_lag1_diff", a["avg_lag1_diff"])
|
|
print("temporal_avg_lag1_diff", b["avg_lag1_diff"])
|
|
print("delta_avg_lag1_diff", f(b["avg_lag1_diff"]) - f(a["avg_lag1_diff"]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|