This commit is contained in:
2026-01-27 18:39:24 +08:00
parent c46c25d607
commit a24c60c506
22 changed files with 357 additions and 8 deletions

View File

@@ -173,6 +173,9 @@ def main():
std = stats["std"]
transforms = stats.get("transform", {})
raw_std = stats.get("raw_std", std)
quantile_probs = stats.get("quantile_probs")
quantile_values = stats.get("quantile_values")
use_quantile = bool(config.get("use_quantile_transform", False))
vocab = load_json(config["vocab_path"])["vocab"]
vocab_sizes = [len(vocab[c]) for c in disc_cols]
@@ -253,6 +256,9 @@ def main():
max_batches=int(config["max_batches"]),
return_file_id=False,
transforms=transforms,
quantile_probs=quantile_probs,
quantile_values=quantile_values,
use_quantile=use_quantile,
shuffle_buffer=int(config.get("shuffle_buffer", 0)),
)
):
@@ -284,6 +290,9 @@ def main():
max_batches=int(config["max_batches"]),
return_file_id=use_condition,
transforms=transforms,
quantile_probs=quantile_probs,
quantile_values=quantile_values,
use_quantile=use_quantile,
shuffle_buffer=int(config.get("shuffle_buffer", 0)),
)
):