update

example/results/config_used.json

@@ -32,6 +32,9 @@
   "model_ff_mult": 2,
   "model_pos_dim": 64,
   "model_use_pos_embed": true,
+  "model_use_feature_graph": true,
+  "feature_graph_scale": 0.1,
+  "feature_graph_dropout": 0.0,
   "disc_mask_scale": 0.9,
   "shuffle_buffer": 256,
   "cont_loss_weighting": "inv_std",
@@ -46,6 +49,9 @@
     0.75,
     0.95
   ],
+  "quantile_loss_warmup_steps": 200,
+  "quantile_loss_clip": 6.0,
+  "quantile_loss_huber_delta": 1.0,
   "sample_batch_size": 8,
   "sample_seq_len": 128
 }

example/results/train_log.csv

@@ -1,61 +1,61 @@
-epoch,step,loss,loss_cont,loss_disc
-0,0,11527.787109,16467.474609,1.724242
-0,10,8925.303711,12749.924805,1.066143
-0,20,122902.148438,175573.968750,1.048266
-0,30,27893.091797,39846.773438,1.019252
-0,40,19384.621094,27692.042969,0.490717
-0,50,27559.511719,39370.480469,0.440001
-1,0,983913.437500,1405590.250000,0.672650
-1,10,15702.782227,22431.642578,1.949623
-1,20,119934.757812,171335.171875,0.266383
-1,30,46955.992188,67079.718750,0.451367
-1,40,14945.393555,21350.371094,0.329458
-1,50,9970.682617,14243.659180,0.292640
-2,0,909690.937500,1299558.000000,0.753066
-2,10,24938.431641,35625.917969,0.833421
-2,20,133833.140625,191190.000000,0.250991
-2,30,84536.242188,120765.851562,0.342500
-2,40,28339.226562,40484.390625,0.374297
-2,50,12867.059570,18381.312500,0.347191
-3,0,992322.187500,1417602.625000,0.799355
-3,10,112537.000000,160766.156250,2.107298
-3,20,92161.367188,131658.921875,0.221603
-3,30,58064.828125,82949.531250,0.354935
-3,40,11155.012695,15935.555664,0.291318
-3,50,8351.337891,11930.288086,0.319319
-4,0,985018.437500,1407168.625000,0.989894
-4,10,150587.375000,215124.531250,0.410451
-4,20,290905.875000,415579.625000,0.302090
-4,30,110148.171875,157354.265625,0.438021
-4,40,22988.332031,32840.261719,0.350128
-4,50,17114.314453,24448.830078,0.326380
-5,0,748518.937500,1069312.250000,0.816294
-5,10,117673.734375,168105.093750,0.387743
-5,20,320272.375000,457531.718750,0.386736
-5,30,31867.970703,45525.425781,0.396934
-5,40,15146.457031,21637.613281,0.302920
-5,50,10223.821289,14605.273438,0.319003
-6,0,593388.375000,847697.375000,0.711933
-6,10,28501.130859,40715.085938,1.744874
-6,20,211790.203125,302557.187500,0.308398
-6,30,16399.605469,23427.794922,0.357711
-6,40,11942.860352,17061.058594,0.288670
-6,50,8927.477539,12753.364258,0.288999
-7,0,535968.187500,765668.375000,0.773959
-7,10,12762.362305,18231.421875,1.077526
-7,20,42435.332031,60621.671875,0.316750
-7,30,15629.781250,22328.056641,0.334986
-7,40,10731.666992,15330.766602,0.313070
-7,50,8851.616211,12644.987305,0.279095
-8,0,365685.375000,522407.281250,0.723811
-8,10,10646.472656,15208.710938,1.137090
-8,20,30592.496094,43703.382812,0.223082
-8,30,26734.046875,38191.308594,0.311048
-8,40,11779.257812,16827.343750,0.270093
-8,50,9673.774414,13819.486328,0.311987
-9,0,126674.117188,180962.640625,0.731923
-9,10,8981.769531,12830.742188,0.734161
-9,20,23218.923828,33169.687500,0.278144
-9,30,19692.558594,28132.021484,0.300332
-9,40,12912.096680,18445.681641,0.284465
-9,50,9004.684570,12863.654297,0.308479
+epoch,step,loss,loss_cont,loss_disc,loss_quantile
+0,0,11527.750000,16467.474609,1.724242,0.113109
+0,10,8955.048828,12792.469727,1.065955,0.142707
+0,20,123049.507812,175784.562500,1.048191,0.181546
+0,30,30947.671875,44210.511719,1.034107,0.224363
+0,40,11166.339844,15951.691406,0.512211,0.163978
+0,50,14919.276367,21313.089844,0.369016,0.101041
+1,0,1113425.000000,1590606.875000,0.667305,0.236102
+1,10,38804.527344,55433.980469,2.453400,0.238323
+1,20,138075.984375,197251.281250,0.288013,0.157941
+1,30,56904.078125,81291.343750,0.429261,0.174654
+1,40,10662.019531,15231.303711,0.343701,0.079876
+1,50,9890.013672,14128.458008,0.292848,0.088391
+2,0,912398.500000,1303426.125000,0.795286,0.256503
+2,10,12546.185547,17922.542969,1.328752,0.110444
+2,20,109676.710938,156680.906250,0.245523,0.132549
+2,30,49427.507812,70610.578125,0.327188,0.088591
+2,40,27778.673828,39683.660156,0.345683,0.102361
+2,50,10311.509766,14730.566406,0.350199,0.091674
+3,0,1040308.062500,1486154.000000,0.807157,0.279805
+3,10,64799.246094,92570.117188,0.485949,0.198213
+3,20,336018.000000,480025.531250,0.410466,0.118048
+3,30,94216.312500,134594.562500,0.355044,0.114209
+3,40,19988.919922,28555.457031,0.298066,0.094291
+3,50,9181.969727,13116.940430,0.326489,0.137392
+4,0,741176.187500,1058822.750000,0.850633,0.233118
+4,10,39252.617188,56074.410156,1.690607,0.227370
+4,20,108992.304688,155703.140625,0.258920,0.285944
+4,30,37115.253906,53021.632812,0.337674,0.131537
+4,40,19358.708984,27655.123047,0.349937,0.169446
+4,50,11434.291992,16334.540039,0.351904,0.089116
+5,0,845658.312500,1208083.000000,0.922483,0.285983
+5,10,130569.406250,186527.515625,0.405198,0.227639
+5,20,245780.390625,351114.687500,0.301236,0.147030
+5,30,42017.671875,60025.066406,0.372036,0.117895
+5,40,11496.740234,16423.779297,0.286911,0.085701
+5,50,8891.728516,12702.317383,0.322913,0.099181
+6,0,617909.687500,882727.812500,0.834663,0.205251
+6,10,18171.734375,25959.302734,0.677465,0.190570
+6,20,423716.187500,605308.687500,0.464189,0.156125
+6,30,48133.507812,68761.914062,0.478786,0.221465
+6,40,20350.281250,29071.666016,0.343182,0.118739
+6,50,11372.219727,16245.889648,0.289177,0.101218
+7,0,534133.500000,763047.500000,0.732465,0.239962
+7,10,19025.574219,27178.703125,1.551094,0.183023
+7,20,58042.757812,82918.078125,0.298854,0.146532
+7,30,11810.656250,16872.201172,0.353035,0.092791
+7,40,9733.656250,13905.077148,0.312043,0.089972
+7,50,9069.159180,12955.803711,0.280795,0.129749
+8,0,650366.937500,929095.250000,0.787194,0.263791
+8,10,13121.248047,18744.031250,1.376233,0.141053
+8,20,19326.507812,27609.167969,0.243424,0.183426
+8,30,12904.376953,18434.667969,0.334577,0.092547
+8,40,9682.833984,13832.478516,0.296229,0.109518
+8,50,8866.144531,12665.773438,0.312481,0.099564
+9,0,265689.562500,379556.156250,0.738840,0.186122
+9,10,9944.607422,14206.222656,0.806142,0.096763
+9,20,22091.175781,31558.679688,0.285403,0.131950
+9,30,19346.830078,27638.111328,0.460825,0.131496
+9,40,10279.702148,14685.139648,0.314872,0.095758
+9,50,10420.340820,14886.050781,0.314008,0.112904