zhicao commited on
Commit
7760e3a
·
verified ·
1 Parent(s): 3e2a7d5

Scheduled Commit

Browse files
Files changed (1) hide show
  1. ablation_runs/eval_fullmodel.json +488 -0
ablation_runs/eval_fullmodel.json ADDED
@@ -0,0 +1,488 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fullmodel": {
3
+ "checkpoint": "checkpoints/trex_fullmodel_wan22_lora/checkpoint-3000",
4
+ "use_track": true,
5
+ "use_force": true,
6
+ "use_deform_tactile": false,
7
+ "forward_tau_losses": {
8
+ "0.8": {
9
+ "action_loss": [
10
+ 0.13547770182291666,
11
+ 0.02183113202980732
12
+ ],
13
+ "dynamics_loss": [
14
+ 0.2144775390625,
15
+ 0.029018606448717087
16
+ ],
17
+ "track_flow_loss": [
18
+ 0.018933614095052082,
19
+ 0.005321941866539868
20
+ ],
21
+ "force_loss": [
22
+ 0.78515625,
23
+ 0.04874993739979955
24
+ ]
25
+ },
26
+ "0.6": {
27
+ "action_loss": [
28
+ 0.20585123697916666,
29
+ 0.02937946686927419
30
+ ],
31
+ "dynamics_loss": [
32
+ 0.2804361979166667,
33
+ 0.028210188338396152
34
+ ],
35
+ "track_flow_loss": [
36
+ 0.0244903564453125,
37
+ 0.008538947969025141
38
+ ],
39
+ "force_loss": [
40
+ 0.78515625,
41
+ 0.04874993739979955
42
+ ]
43
+ },
44
+ "0.4": {
45
+ "action_loss": [
46
+ 0.400634765625,
47
+ 0.042702747382848516
48
+ ],
49
+ "dynamics_loss": [
50
+ 0.4158528645833333,
51
+ 0.0261730693588898
52
+ ],
53
+ "track_flow_loss": [
54
+ 0.050028483072916664,
55
+ 0.017132937881065836
56
+ ],
57
+ "force_loss": [
58
+ 0.78515625,
59
+ 0.04874993739979955
60
+ ]
61
+ },
62
+ "0.2": {
63
+ "action_loss": [
64
+ 0.748046875,
65
+ 0.04617808756850347
66
+ ],
67
+ "dynamics_loss": [
68
+ 0.72802734375,
69
+ 0.021626321447198667
70
+ ],
71
+ "track_flow_loss": [
72
+ 0.2729085286458333,
73
+ 0.028166929238128222
74
+ ],
75
+ "force_loss": [
76
+ 0.78515625,
77
+ 0.04874993739979955
78
+ ]
79
+ },
80
+ "0.05": {
81
+ "action_loss": [
82
+ 1.0813802083333333,
83
+ 0.05201411415982778
84
+ ],
85
+ "dynamics_loss": [
86
+ 1.1334635416666667,
87
+ 0.015282154419668624
88
+ ],
89
+ "track_flow_loss": [
90
+ 0.9150390625,
91
+ 0.02557152106574588
92
+ ],
93
+ "force_loss": [
94
+ 0.78515625,
95
+ 0.04874993739979955
96
+ ]
97
+ }
98
+ },
99
+ "openloop_action_mse": {
100
+ "cascade": [
101
+ 0.2360212141647935,
102
+ 0.09099028876810561,
103
+ 16
104
+ ],
105
+ "force_future": [
106
+ 0.23825530987232924,
107
+ 0.08397315237613685,
108
+ 16
109
+ ],
110
+ "coarse_only": [
111
+ 0.38328345865011215,
112
+ 0.08656744394776938,
113
+ 16
114
+ ]
115
+ },
116
+ "openloop_action_mse_raw": {
117
+ "cascade": [
118
+ 0.2825452983379364,
119
+ 0.19788438081741333,
120
+ 0.18749985098838806,
121
+ 0.19937412440776825,
122
+ 0.17939801514148712,
123
+ 0.18645183742046356,
124
+ 0.21017836034297943,
125
+ 0.19251590967178345,
126
+ 0.18035215139389038,
127
+ 0.2854746878147125,
128
+ 0.1778016984462738,
129
+ 0.19986674189567566,
130
+ 0.2286730855703354,
131
+ 0.22805914282798767,
132
+ 0.28089261054992676,
133
+ 0.5593715310096741
134
+ ],
135
+ "force_future": [
136
+ 0.2653614282608032,
137
+ 0.17910310626029968,
138
+ 0.2507321536540985,
139
+ 0.20312190055847168,
140
+ 0.2078472077846527,
141
+ 0.21611593663692474,
142
+ 0.19994448125362396,
143
+ 0.17686443030834198,
144
+ 0.21135500073432922,
145
+ 0.22447068989276886,
146
+ 0.18521061539649963,
147
+ 0.22188623249530792,
148
+ 0.25031328201293945,
149
+ 0.27529338002204895,
150
+ 0.20027852058410645,
151
+ 0.5441865921020508
152
+ ],
153
+ "coarse_only": [
154
+ 0.4576667547225952,
155
+ 0.3609382212162018,
156
+ 0.32956430315971375,
157
+ 0.3707992732524872,
158
+ 0.3335039019584656,
159
+ 0.31421878933906555,
160
+ 0.3521805703639984,
161
+ 0.3446769416332245,
162
+ 0.3028632700443268,
163
+ 0.3965727686882019,
164
+ 0.33890533447265625,
165
+ 0.32683470845222473,
166
+ 0.34397807717323303,
167
+ 0.412160187959671,
168
+ 0.4892078638076782,
169
+ 0.6584643721580505
170
+ ]
171
+ },
172
+ "openloop_action_mse_split": {
173
+ "cascade_arm": 0.31711942330002785,
174
+ "cascade_hand": 0.20284467656165361,
175
+ "coarse_arm": 0.4167211055755615,
176
+ "coarse_hand": 0.36960443295538425
177
+ },
178
+ "stage1_attention": {
179
+ "0": {
180
+ "action": {
181
+ "cond_obs": 0.031286440789699554,
182
+ "obs": 0.11673178523778915,
183
+ "action": 0.05415532737970352,
184
+ "state": 0.0018348912708461285,
185
+ "track_past": 0.45700106024742126,
186
+ "track_future": 0.3389905095100403
187
+ },
188
+ "obs": {
189
+ "cond_obs": 0.05332382023334503,
190
+ "obs": 0.5820091366767883,
191
+ "action": 0.010074972175061703,
192
+ "state": 0.0005240149330347776,
193
+ "track_past": 0.17835120856761932,
194
+ "track_future": 0.17571686208248138
195
+ }
196
+ },
197
+ "14": {
198
+ "action": {
199
+ "cond_obs": 0.05233272165060043,
200
+ "obs": 0.10831200331449509,
201
+ "action": 0.22056667506694794,
202
+ "state": 0.0014698345912620425,
203
+ "track_past": 0.5930889248847961,
204
+ "track_future": 0.0242298673838377
205
+ },
206
+ "obs": {
207
+ "cond_obs": 0.16479282081127167,
208
+ "obs": 0.6339880228042603,
209
+ "action": 0.014906498603522778,
210
+ "state": 0.00244765542447567,
211
+ "track_past": 0.16708970069885254,
212
+ "track_future": 0.016775289550423622
213
+ }
214
+ },
215
+ "29": {
216
+ "action": {
217
+ "cond_obs": 0.05360835790634155,
218
+ "obs": 0.0986405462026596,
219
+ "action": 0.2380063831806183,
220
+ "state": 0.0021925801411271095,
221
+ "track_past": 0.5321187973022461,
222
+ "track_future": 0.0754334032535553
223
+ },
224
+ "obs": {
225
+ "cond_obs": 0.09494271874427795,
226
+ "obs": 0.7751515507698059,
227
+ "action": 0.010304520837962627,
228
+ "state": 0.00035713627585209906,
229
+ "track_past": 0.07586673647165298,
230
+ "track_future": 0.04337741434574127
231
+ }
232
+ }
233
+ }
234
+ },
235
+ "full": {
236
+ "checkpoint": "checkpoints/ablate_full_3k/checkpoint-3000",
237
+ "use_track": true,
238
+ "use_force": true,
239
+ "use_deform_tactile": false,
240
+ "forward_tau_losses": {
241
+ "0.8": {
242
+ "action_loss": [
243
+ 0.13885498046875,
244
+ 0.021893762307867347
245
+ ],
246
+ "dynamics_loss": [
247
+ 0.214599609375,
248
+ 0.028883325232585192
249
+ ],
250
+ "track_flow_loss": [
251
+ 0.01959228515625,
252
+ 0.005815500664137739
253
+ ],
254
+ "force_loss": [
255
+ 0.6427408854166666,
256
+ 0.07174448889029805
257
+ ]
258
+ },
259
+ "0.6": {
260
+ "action_loss": [
261
+ 0.20402018229166666,
262
+ 0.027474107787100906
263
+ ],
264
+ "dynamics_loss": [
265
+ 0.2801106770833333,
266
+ 0.02749254221890766
267
+ ],
268
+ "track_flow_loss": [
269
+ 0.025583902994791668,
270
+ 0.009258714454018649
271
+ ],
272
+ "force_loss": [
273
+ 0.6427408854166666,
274
+ 0.07174448889029805
275
+ ]
276
+ },
277
+ "0.4": {
278
+ "action_loss": [
279
+ 0.3986002604166667,
280
+ 0.0390743661447396
281
+ ],
282
+ "dynamics_loss": [
283
+ 0.4138997395833333,
284
+ 0.025458871788911118
285
+ ],
286
+ "track_flow_loss": [
287
+ 0.051778157552083336,
288
+ 0.017947368985419505
289
+ ],
290
+ "force_loss": [
291
+ 0.6427408854166666,
292
+ 0.07174448889029805
293
+ ]
294
+ },
295
+ "0.2": {
296
+ "action_loss": [
297
+ 0.7521158854166666,
298
+ 0.048475014437045466
299
+ ],
300
+ "dynamics_loss": [
301
+ 0.7236328125,
302
+ 0.020585171554378095
303
+ ],
304
+ "track_flow_loss": [
305
+ 0.270751953125,
306
+ 0.02853870589963502
307
+ ],
308
+ "force_loss": [
309
+ 0.6427408854166666,
310
+ 0.07174448889029805
311
+ ]
312
+ },
313
+ "0.05": {
314
+ "action_loss": [
315
+ 1.0758463541666667,
316
+ 0.05334471104376999
317
+ ],
318
+ "dynamics_loss": [
319
+ 1.1305338541666667,
320
+ 0.01504103522415094
321
+ ],
322
+ "track_flow_loss": [
323
+ 0.912109375,
324
+ 0.026373214974647258
325
+ ],
326
+ "force_loss": [
327
+ 0.6427408854166666,
328
+ 0.07174448889029805
329
+ ]
330
+ }
331
+ },
332
+ "openloop_action_mse": {
333
+ "cascade": [
334
+ 0.20543859526515007,
335
+ 0.10553651615425272,
336
+ 16
337
+ ],
338
+ "coarse_only": [
339
+ 0.3859300408512354,
340
+ 0.09367721956997326,
341
+ 16
342
+ ]
343
+ },
344
+ "openloop_action_mse_raw": {
345
+ "cascade": [
346
+ 0.2599409520626068,
347
+ 0.16157826781272888,
348
+ 0.15067900717258453,
349
+ 0.15890291333198547,
350
+ 0.13666832447052002,
351
+ 0.14541499316692352,
352
+ 0.1762809008359909,
353
+ 0.1548680067062378,
354
+ 0.14876900613307953,
355
+ 0.2613345682621002,
356
+ 0.14742904901504517,
357
+ 0.15507377684116364,
358
+ 0.17896708846092224,
359
+ 0.19943304359912872,
360
+ 0.27371665835380554,
361
+ 0.5779609680175781
362
+ ],
363
+ "coarse_only": [
364
+ 0.44467893242836,
365
+ 0.35982707142829895,
366
+ 0.32907775044441223,
367
+ 0.36753952503204346,
368
+ 0.33504316210746765,
369
+ 0.3160090148448944,
370
+ 0.34967127442359924,
371
+ 0.3433721661567688,
372
+ 0.2978774905204773,
373
+ 0.3976256549358368,
374
+ 0.34054020047187805,
375
+ 0.3271501660346985,
376
+ 0.3467199206352234,
377
+ 0.42348822951316833,
378
+ 0.5095632076263428,
379
+ 0.6866968870162964
380
+ ]
381
+ },
382
+ "openloop_action_mse_split": {
383
+ "cascade_arm": 0.29095884039998055,
384
+ "cascade_hand": 0.1704530455172062,
385
+ "coarse_arm": 0.42187667451798916,
386
+ "coarse_hand": 0.3712246026843786
387
+ },
388
+ "stage1_attention": {
389
+ "0": {
390
+ "action": {
391
+ "cond_obs": 0.028233155608177185,
392
+ "obs": 0.10685919225215912,
393
+ "action": 0.06159074231982231,
394
+ "state": 0.001643829746171832,
395
+ "track_past": 0.4548226296901703,
396
+ "track_future": 0.3468504846096039
397
+ },
398
+ "obs": {
399
+ "cond_obs": 0.053328514099121094,
400
+ "obs": 0.5826746821403503,
401
+ "action": 0.009129117242991924,
402
+ "state": 0.0005130452918820083,
403
+ "track_past": 0.1784334033727646,
404
+ "track_future": 0.1759212762117386
405
+ }
406
+ },
407
+ "14": {
408
+ "action": {
409
+ "cond_obs": 0.04729510098695755,
410
+ "obs": 0.08689284324645996,
411
+ "action": 0.20308324694633484,
412
+ "state": 0.0032673801761120558,
413
+ "track_past": 0.6379822492599487,
414
+ "track_future": 0.021479224786162376
415
+ },
416
+ "obs": {
417
+ "cond_obs": 0.16180963814258575,
418
+ "obs": 0.6352365016937256,
419
+ "action": 0.013975701294839382,
420
+ "state": 0.006121945567429066,
421
+ "track_past": 0.167484849691391,
422
+ "track_future": 0.015371335670351982
423
+ }
424
+ },
425
+ "29": {
426
+ "action": {
427
+ "cond_obs": 0.05335542559623718,
428
+ "obs": 0.09940896928310394,
429
+ "action": 0.18759986758232117,
430
+ "state": 0.0025236550718545914,
431
+ "track_past": 0.5753425359725952,
432
+ "track_future": 0.08176963031291962
433
+ },
434
+ "obs": {
435
+ "cond_obs": 0.09543927758932114,
436
+ "obs": 0.7705966234207153,
437
+ "action": 0.01101355254650116,
438
+ "state": 0.0004360256134532392,
439
+ "track_past": 0.07644421607255936,
440
+ "track_future": 0.046070314943790436
441
+ }
442
+ }
443
+ },
444
+ "stage2_attention": [
445
+ {
446
+ "action": 0.033935546875,
447
+ "f6": 0.0230712890625,
448
+ "vq": 0.024169921875,
449
+ "memory": 0.91796875
450
+ },
451
+ {
452
+ "action": 0.0322265625,
453
+ "f6": 0.0208740234375,
454
+ "vq": 0.023681640625,
455
+ "memory": 0.921875
456
+ },
457
+ {
458
+ "action": 0.0274658203125,
459
+ "f6": 0.01708984375,
460
+ "vq": 0.0203857421875,
461
+ "memory": 0.93359375
462
+ },
463
+ {
464
+ "action": 0.0250244140625,
465
+ "f6": 0.01513671875,
466
+ "vq": 0.0185546875,
467
+ "memory": 0.94140625
468
+ },
469
+ {
470
+ "action": 0.024169921875,
471
+ "f6": 0.01416015625,
472
+ "vq": 0.017333984375,
473
+ "memory": 0.9453125
474
+ },
475
+ {
476
+ "action": 0.0228271484375,
477
+ "f6": 0.0130615234375,
478
+ "vq": 0.0157470703125,
479
+ "memory": 0.94921875
480
+ }
481
+ ],
482
+ "tactile_sensitivity": {
483
+ "mean_abs_delta_vs_masked": 0.014084354043006897,
484
+ "max_abs_delta_vs_masked": 0.06640625,
485
+ "mean_abs_refinement": 0.2452070713043213
486
+ }
487
+ }
488
+ }