{
 "schema": "coreai-decider-readout-gate/1",
 "bundle": "/private/tmp/claude-501/-Users-majimadaisuke-code-coreai/435735e2-59de-4af7-8bc7-8d80e7bc59a1/scratchpad/hfstage/gpu-pipelined/decider_0_8b_decode_int8hu_block32_sym",
 "asset": "/private/tmp/claude-501/-Users-majimadaisuke-code-coreai/435735e2-59de-4af7-8bc7-8d80e7bc59a1/scratchpad/ship-aotc/decider_0_8b_decode_int8hu_block32_sym.h16c.aimodelc",
 "aot_seconds": 36.893706707982346,
 "fixtures": "models/decider-0.8b/fixtures-decider-0.8b.json",
 "temperature": 1.03,
 "max_ctx": 4096,
 "runtime": "coreai python runtime, AOT h16c GPU asset, SpecializationOptions.default()",
 "environment": {
  "platform": "macOS-27.0-arm64-arm-64bit",
  "python": "3.11.13"
 },
 "rows": [
  {
   "id": "r01-route",
   "tokens": 67,
   "nopts": 3,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9723106026649475,
    0.02710120379924774,
    0.0005881211836822331
   ],
   "p_oracle": [
    0.9706390500068665,
    0.02876637876033783,
    0.0005945737357251346
   ],
   "oracle_margin": 0.9418726563453674,
   "max_abs_delta_p": 0.0016715526580810547,
   "mean_abs_delta_p": 0.0011143933904046814,
   "finite": true,
   "nonconstant": true,
   "seconds": 2.4876702079782262
  },
  {
   "id": "r01-condition",
   "tokens": 61,
   "nopts": 4,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.0019042574567720294,
    0.9939403533935547,
    0.0007663607248105109,
    0.0033890451304614544
   ],
   "p_oracle": [
    0.0018363530980423093,
    0.9941917061805725,
    0.0007545964326709509,
    0.0032173478975892067
   ],
   "oracle_margin": 0.9909743666648865,
   "max_abs_delta_p": 0.00025135278701782227,
   "mean_abs_delta_p": 0.00012567966768983752,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5768512499053031
  },
  {
   "id": "r01-refund",
   "tokens": 63,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9972710013389587,
    0.002728985622525215
   ],
   "p_oracle": [
    0.9973737001419067,
    0.0026263452600687742
   ],
   "oracle_margin": 0.9947473406791687,
   "max_abs_delta_p": 0.00010269880294799805,
   "mean_abs_delta_p": 0.00010266958270221949,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5607595829060301
  },
  {
   "id": "r02-action",
   "tokens": 67,
   "nopts": 3,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.0008631355012767017,
    0.9990667700767517,
    7.010211993474513e-05
   ],
   "p_oracle": [
    0.0008344926754944026,
    0.999098539352417,
    6.697019125567749e-05
   ],
   "oracle_margin": 0.9982640743255615,
   "max_abs_delta_p": 3.17692756652832e-05,
   "mean_abs_delta_p": 2.118134337554996e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5979873749893159
  },
  {
   "id": "r02-lamp",
   "tokens": 62,
   "nopts": 3,
   "letter_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 34,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.11018924415111542,
    0.0026190667413175106,
    0.887191653251648
   ],
   "p_oracle": [
    0.10940099507570267,
    0.002520693466067314,
    0.8880782723426819
   ],
   "oracle_margin": 0.7786772847175598,
   "max_abs_delta_p": 0.0008866190910339355,
   "mean_abs_delta_p": 0.0005910804805656275,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5741835839580745
  },
  {
   "id": "r02-cold",
   "tokens": 58,
   "nopts": 2,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.039133764803409576,
    0.9608662128448486
   ],
   "p_oracle": [
    0.04125106334686279,
    0.9587488770484924
   ],
   "oracle_margin": 0.9174978137016296,
   "max_abs_delta_p": 0.002117335796356201,
   "mean_abs_delta_p": 0.002117317169904709,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5285169170238078
  },
  {
   "id": "r03-color",
   "tokens": 61,
   "nopts": 6,
   "letter_argmax": 4,
   "oracle_argmax": 4,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 36,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.011212646029889584,
    0.0069005792029201984,
    0.024677135050296783,
    0.005929300561547279,
    0.9407278895378113,
    0.010552494786679745
   ],
   "p_oracle": [
    0.011068147607147694,
    0.006971844006329775,
    0.025079987943172455,
    0.005784797016531229,
    0.9406063556671143,
    0.010488779284060001
   ],
   "oracle_margin": 0.9155263900756836,
   "max_abs_delta_p": 0.0004028528928756714,
   "mean_abs_delta_p": 0.00015806150622665882,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5598345830803737
  },
  {
   "id": "r03-shape",
   "tokens": 71,
   "nopts": 8,
   "letter_argmax": 4,
   "oracle_argmax": 4,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 36,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.004042176529765129,
    0.006275858264416456,
    0.008436127565801144,
    0.009892802685499191,
    0.9586568474769592,
    0.002544924383983016,
    0.005152599886059761,
    0.004998619202524424
   ],
   "p_oracle": [
    0.003979754168540239,
    0.006537850946187973,
    0.008193938992917538,
    0.00970080029219389,
    0.9587033987045288,
    0.0025064260698854923,
    0.005255395546555519,
    0.005122386384755373
   ],
   "oracle_margin": 0.9490026235580444,
   "max_abs_delta_p": 0.0002619926817715168,
   "mean_abs_delta_p": 0.00013377729919739068,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6880730000557378
  },
  {
   "id": "r03-wet",
   "tokens": 56,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.996523916721344,
    0.003476059529930353
   ],
   "p_oracle": [
    0.9964544773101807,
    0.003545538056641817
   ],
   "oracle_margin": 0.9929089546203613,
   "max_abs_delta_p": 6.947852671146393e-05,
   "mean_abs_delta_p": 6.9458968937397e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5228670419892296
  },
  {
   "id": "r04-drawer",
   "tokens": 74,
   "nopts": 7,
   "letter_argmax": 6,
   "oracle_argmax": 6,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 38,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.011951428838074207,
    0.002235749736428261,
    0.0012094862759113312,
    0.0014956698287278414,
    0.001727522350847721,
    0.0012279736110940576,
    0.9801521897315979
   ],
   "p_oracle": [
    0.011879051104187965,
    0.00227216980420053,
    0.0012058026622980833,
    0.0014660466695204377,
    0.001746279769577086,
    0.0012176926247775555,
    0.9802129864692688
   ],
   "oracle_margin": 0.9683339595794678,
   "max_abs_delta_p": 7.237773388624191e-05,
   "mean_abs_delta_p": 3.3134245313704014e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6362709579989314
  },
  {
   "id": "r04-material",
   "tokens": 89,
   "nopts": 10,
   "letter_argmax": 5,
   "oracle_argmax": 5,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 37,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.002266624476760626,
    0.007686525583267212,
    0.0266663059592247,
    0.002824285300448537,
    0.0020694276317954063,
    0.942304253578186,
    0.002520559588447213,
    0.006030018907040358,
    0.0014162667794153094,
    0.006215774454176426
   ],
   "p_oracle": [
    0.002213399624451995,
    0.007533108815550804,
    0.027071012184023857,
    0.002823340008035302,
    0.002065989887341857,
    0.9418793320655823,
    0.002517657820135355,
    0.006125518586486578,
    0.0014496404910460114,
    0.006320958025753498
   ],
   "oracle_margin": 0.9148083329200745,
   "max_abs_delta_p": 0.00042492151260375977,
   "mean_abs_delta_p": 0.00012776111252605916,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.7990567079978064
  },
  {
   "id": "r04-key",
   "tokens": 54,
   "nopts": 2,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.017900662496685982,
    0.9820992946624756
   ],
   "p_oracle": [
    0.01657148450613022,
    0.9834285378456116
   ],
   "oracle_margin": 0.9668570756912231,
   "max_abs_delta_p": 0.0013292431831359863,
   "mean_abs_delta_p": 0.0013292105868458748,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.49435312498826534
  },
  {
   "id": "r05-task",
   "tokens": 50,
   "nopts": 3,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.0043908897787332535,
    0.9950297474861145,
    0.0005794651224277914
   ],
   "p_oracle": [
    0.004327560309320688,
    0.9951086640357971,
    0.0005636969581246376
   ],
   "oracle_margin": 0.9907811284065247,
   "max_abs_delta_p": 7.891654968261719e-05,
   "mean_abs_delta_p": 5.2671394466112055e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.46803537500090897
  },
  {
   "id": "r05-leaves",
   "tokens": 54,
   "nopts": 4,
   "letter_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 34,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.0006353064673021436,
    0.0024140458554029465,
    0.9960081577301025,
    0.0009424899471923709
   ],
   "p_oracle": [
    0.0006024409085512161,
    0.0023958953097462654,
    0.9961035251617432,
    0.0008982465951703489
   ],
   "oracle_margin": 0.9937076568603516,
   "max_abs_delta_p": 9.5367431640625e-05,
   "mean_abs_delta_p": 4.765672201756388e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.4758549160324037
  },
  {
   "id": "r05-dry",
   "tokens": 46,
   "nopts": 2,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.021082883700728416,
    0.978917121887207
   ],
   "p_oracle": [
    0.02092888019979,
    0.9790711402893066
   ],
   "oracle_margin": 0.9581422805786133,
   "max_abs_delta_p": 0.00015401840209960938,
   "mean_abs_delta_p": 0.00015401095151901245,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.445729416096583
  },
  {
   "id": "r06-phase",
   "tokens": 62,
   "nopts": 4,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.005169644486159086,
    0.9839650392532349,
    0.009131002239882946,
    0.0017342425417155027
   ],
   "p_oracle": [
    0.0053413608111441135,
    0.9834700226783752,
    0.009449634701013565,
    0.001738991355523467
   ],
   "oracle_margin": 0.9740203619003296,
   "max_abs_delta_p": 0.0004950165748596191,
   "mean_abs_delta_p": 0.00024752854369580746,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5606856249505654
  },
  {
   "id": "r06-priority",
   "tokens": 57,
   "nopts": 3,
   "letter_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 34,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.007412690203636885,
    0.06293796747922897,
    0.9296493530273438
   ],
   "p_oracle": [
    0.007694360800087452,
    0.06237579137086868,
    0.9299298524856567
   ],
   "oracle_margin": 0.8675540685653687,
   "max_abs_delta_p": 0.0005621761083602905,
   "mean_abs_delta_p": 0.00037478205437461537,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5206250000046566
  },
  {
   "id": "r06-approved",
   "tokens": 64,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9920893907546997,
    0.007910564541816711
   ],
   "p_oracle": [
    0.9920409321784973,
    0.007959095761179924
   ],
   "oracle_margin": 0.9840818643569946,
   "max_abs_delta_p": 4.8531219363212585e-05,
   "mean_abs_delta_p": 4.849489778280258e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6062248749658465
  },
  {
   "id": "r07-contents",
   "tokens": 60,
   "nopts": 4,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.004440651275217533,
    0.9836660623550415,
    0.005969206802546978,
    0.0059241014532744884
   ],
   "p_oracle": [
    0.0044456240721046925,
    0.9837862849235535,
    0.005914564710110426,
    0.005853535141795874
   ],
   "oracle_margin": 0.9778717160224915,
   "max_abs_delta_p": 0.00012022256851196289,
   "mean_abs_delta_p": 6.260094232857227e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5552980829961598
  },
  {
   "id": "r07-location",
   "tokens": 69,
   "nopts": 5,
   "letter_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 34,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.003617149079218507,
    0.0020478996448218822,
    0.9908372163772583,
    0.002902934793382883,
    0.0005947985337115824
   ],
   "p_oracle": [
    0.0037419262807816267,
    0.0021192876156419516,
    0.9904859662055969,
    0.00304310186766088,
    0.0006097119185142219
   ],
   "oracle_margin": 0.9867440462112427,
   "max_abs_delta_p": 0.00035125017166137695,
   "mean_abs_delta_p": 0.00014049916062504054,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.7628027080791071
  },
  {
   "id": "r07-seal",
   "tokens": 51,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9954375624656677,
    0.004562480840831995
   ],
   "p_oracle": [
    0.9957252740859985,
    0.004274771548807621
   ],
   "oracle_margin": 0.9914504885673523,
   "max_abs_delta_p": 0.00028771162033081055,
   "mean_abs_delta_p": 0.0002877104561775923,
   "finite": true,
   "nonconstant": true,
   "seconds": 6.538128832937218
  },
  {
   "id": "r08-direction",
   "tokens": 67,
   "nopts": 5,
   "letter_argmax": 3,
   "oracle_argmax": 3,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 35,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.02494763396680355,
    0.06841401010751724,
    0.0743669793009758,
    0.8296688795089722,
    0.0026024768594652414
   ],
   "p_oracle": [
    0.026061559095978737,
    0.07013165950775146,
    0.0797516331076622,
    0.8213180303573608,
    0.002737182192504406
   ],
   "oracle_margin": 0.7415664196014404,
   "max_abs_delta_p": 0.008350849151611328,
   "mean_abs_delta_p": 0.0033403565641492603,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6042522919597104
  },
  {
   "id": "r08-surface",
   "tokens": 58,
   "nopts": 3,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9964460730552673,
    0.000928708235733211,
    0.002625254215672612
   ],
   "p_oracle": [
    0.9964224696159363,
    0.0009224751847796142,
    0.0026550705078989267
   ],
   "oracle_margin": 0.9937673807144165,
   "max_abs_delta_p": 2.9816292226314545e-05,
   "mean_abs_delta_p": 1.9884260836988688e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5385979579295963
  },
  {
   "id": "r08-open",
   "tokens": 62,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9927181005477905,
    0.007281946483999491
   ],
   "p_oracle": [
    0.9926348328590393,
    0.007365191355347633
   ],
   "oracle_margin": 0.9852696657180786,
   "max_abs_delta_p": 8.32676887512207e-05,
   "mean_abs_delta_p": 8.325628004968166e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5786961669800803
  },
  {
   "id": "r09-liquid",
   "tokens": 58,
   "nopts": 3,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.006688621826469898,
    0.9836852550506592,
    0.009626169688999653
   ],
   "p_oracle": [
    0.00639850040897727,
    0.98415607213974,
    0.00944542046636343
   ],
   "oracle_margin": 0.9747106432914734,
   "max_abs_delta_p": 0.00047081708908081055,
   "mean_abs_delta_p": 0.00031389590973655385,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5406022919341922
  },
  {
   "id": "r09-valve",
   "tokens": 57,
   "nopts": 3,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.045177560299634933,
    0.9531297087669373,
    0.00169268564786762
   ],
   "p_oracle": [
    0.045570921152830124,
    0.9527929425239563,
    0.0016362133901566267
   ],
   "oracle_margin": 0.9072220325469971,
   "max_abs_delta_p": 0.00039336085319519043,
   "mean_abs_delta_p": 0.0002621997846290469,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5073714590398595
  },
  {
   "id": "r09-clarity",
   "tokens": 61,
   "nopts": 4,
   "letter_argmax": 3,
   "oracle_argmax": 3,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 35,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.0014003148535266519,
    0.0006608600378967822,
    0.0004256489919498563,
    0.9975131750106812
   ],
   "p_oracle": [
    0.0014170478098094463,
    0.0006711058667860925,
    0.000431353080784902,
    0.9974804520606995
   ],
   "oracle_margin": 0.9960634112358093,
   "max_abs_delta_p": 3.272294998168945e-05,
   "mean_abs_delta_p": 1.6351455997209996e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.559251083061099
  },
  {
   "id": "r09-fill-level0",
   "tokens": 66,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9961651563644409,
    0.00383490021340549
   ],
   "p_oracle": [
    0.9963890314102173,
    0.0036110365763306618
   ],
   "oracle_margin": 0.9927780032157898,
   "max_abs_delta_p": 0.0002238750457763672,
   "mean_abs_delta_p": 0.00022386934142559767,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.615606417064555
  },
  {
   "id": "r09-fill-level1",
   "tokens": 68,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.7614231705665588,
    0.23857688903808594
   ],
   "p_oracle": [
    0.7548943758010864,
    0.24510566890239716
   ],
   "oracle_margin": 0.5097886919975281,
   "max_abs_delta_p": 0.006528794765472412,
   "mean_abs_delta_p": 0.006528787314891815,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6297322090249509
  },
  {
   "id": "r09-fill-level2",
   "tokens": 67,
   "nopts": 2,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.03211956471204758,
    0.9678804278373718
   ],
   "p_oracle": [
    0.03151174634695053,
    0.9684882164001465
   ],
   "oracle_margin": 0.9369764924049377,
   "max_abs_delta_p": 0.0006078183650970459,
   "mean_abs_delta_p": 0.000607803463935852,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6055792080005631
  },
  {
   "id": "r09-fill-level3",
   "tokens": 68,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.8299474716186523,
    0.17005255818367004
   ],
   "p_oracle": [
    0.821812629699707,
    0.17818734049797058
   ],
   "oracle_margin": 0.6436252593994141,
   "max_abs_delta_p": 0.008134841918945312,
   "mean_abs_delta_p": 0.008134812116622925,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6145045829471201
  },
  {
   "id": "r09-fill-level4",
   "tokens": 67,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9944901466369629,
    0.005509854294359684
   ],
   "p_oracle": [
    0.9943530559539795,
    0.0056469617411494255
   ],
   "oracle_margin": 0.988706111907959,
   "max_abs_delta_p": 0.00013710744678974152,
   "mean_abs_delta_p": 0.00013709906488656998,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6057822089642286
  },
  {
   "id": "r10-surface",
   "tokens": 63,
   "nopts": 4,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.006071456708014011,
    0.9929555654525757,
    0.0006937124999240041,
    0.00027918192790821195
   ],
   "p_oracle": [
    0.006152009591460228,
    0.9928968548774719,
    0.0006806378369219601,
    0.00027054722886532545
   ],
   "oracle_margin": 0.9867448210716248,
   "max_abs_delta_p": 8.055288344621658e-05,
   "mean_abs_delta_p": 4.02432051487267e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5903222500346601
  },
  {
   "id": "r10-color",
   "tokens": 58,
   "nopts": 3,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.005220762919634581,
    0.9713388085365295,
    0.023440472781658173
   ],
   "p_oracle": [
    0.005320006050169468,
    0.9705416560173035,
    0.024138320237398148
   ],
   "oracle_margin": 0.9464033246040344,
   "max_abs_delta_p": 0.0007971525192260742,
   "mean_abs_delta_p": 0.0005314143685003122,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5198557080002502
  },
  {
   "id": "r10-count",
   "tokens": 58,
   "nopts": 3,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.01731935702264309,
    0.9359006881713867,
    0.046779897063970566
   ],
   "p_oracle": [
    0.016619298607110977,
    0.9365748763084412,
    0.046805839985609055
   ],
   "oracle_margin": 0.8897690176963806,
   "max_abs_delta_p": 0.0007000584155321121,
   "mean_abs_delta_p": 0.0004667298247416814,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.51173775002826
  },
  {
   "id": "r10-damage-level0",
   "tokens": 68,
   "nopts": 2,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.006216374225914478,
    0.9937835931777954
   ],
   "p_oracle": [
    0.006841752212494612,
    0.9931582808494568
   ],
   "oracle_margin": 0.9863165020942688,
   "max_abs_delta_p": 0.0006253779865801334,
   "mean_abs_delta_p": 0.0006253451574593782,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6095023748930544
  },
  {
   "id": "r10-damage-level1",
   "tokens": 68,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9978578686714172,
    0.0021421287674456835
   ],
   "p_oracle": [
    0.9978333115577698,
    0.0021666535176336765
   ],
   "oracle_margin": 0.9956666827201843,
   "max_abs_delta_p": 2.4557113647460938e-05,
   "mean_abs_delta_p": 2.4540931917726994e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6242967090802267
  },
  {
   "id": "r10-damage-level2",
   "tokens": 69,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9986905455589294,
    0.0013094566529616714
   ],
   "p_oracle": [
    0.9986666440963745,
    0.0013333152746781707
   ],
   "oracle_margin": 0.9973333477973938,
   "max_abs_delta_p": 2.390146255493164e-05,
   "mean_abs_delta_p": 2.3880042135715485e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6325680409790948
  },
  {
   "id": "r10-damage-level3",
   "tokens": 68,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9984763264656067,
    0.0015236317412927747
   ],
   "p_oracle": [
    0.9985060691833496,
    0.0014939777320250869
   ],
   "oracle_margin": 0.9970120787620544,
   "max_abs_delta_p": 2.9742717742919922e-05,
   "mean_abs_delta_p": 2.969836350530386e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.5561036670114845
  },
  {
   "id": "r10-damage-level4",
   "tokens": 68,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9981865286827087,
    0.0018135028658434749
   ],
   "p_oracle": [
    0.9982340335845947,
    0.001765925670042634
   ],
   "oracle_margin": 0.9964681267738342,
   "max_abs_delta_p": 4.7577195800840855e-05,
   "mean_abs_delta_p": 4.754104884341359e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.6869550830451772
  },
  {
   "id": "r11-bin",
   "tokens": 87,
   "nopts": 11,
   "letter_argmax": 6,
   "oracle_argmax": 6,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 38,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.0009616761817596853,
    0.0005527890170924366,
    0.0004785986675415188,
    0.0005698174936696887,
    0.001119208405725658,
    0.0012540719471871853,
    0.9934198260307312,
    0.0004822426417376846,
    0.0004436397575773299,
    0.0003754569042939693,
    0.00034279224928468466
   ],
   "p_oracle": [
    0.0008766235550865531,
    0.0005157003179192543,
    0.0004343294713180512,
    0.0005023173289373517,
    0.0010404207278043032,
    0.0011031980393454432,
    0.9940218329429626,
    0.00044398775207810104,
    0.0003984920331276953,
    0.00034754970693029463,
    0.00031549902632832527
   ],
   "oracle_margin": 0.9929186105728149,
   "max_abs_delta_p": 0.0006020069122314453,
   "mean_abs_delta_p": 0.00010947111083871938,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.7995729160029441
  },
  {
   "id": "r12-slot",
   "tokens": 1965,
   "nopts": 255,
   "letter_argmax": 172,
   "oracle_argmax": 172,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 39500,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.0002153891691705212,
    4.106413325644098e-05,
    0.00026838164194487035,
    0.0020030417945235968,
    2.6650091967894696e-05,
    3.692708560265601e-05,
    0.009628736414015293,
    8.318667823914438e-06,
    5.294375660014339e-05,
    1.6215666619245894e-05,
    5.561940997722559e-05,
    0.0001219456025864929,
    0.00029844907112419605,
    0.00017818505875766277,
    0.0002525805321056396,
    0.0004776444402523339,
    0.10580489039421082,
    0.0005075250519439578,
    0.00010320381261408329,
    3.7349618651205674e-05,
    8.574925232096575e-06,
    2.0052550098625943e-05,
    1.1881648788403254e-05,
    9.494253026787192e-05,
    3.089822712354362e-05,
    2.3874501493992284e-05,
    7.174950042099226e-06,
    1.1836669727927074e-05,
    2.9523585908464156e-05,
    0.000863067340105772,
    3.501759056234732e-05,
    2.7175260584044736e-06,
    0.00038043601671233773,
    2.4717082851566374e-06,
    3.0433035135501996e-05,
    1.0061120292448322e-06,
    4.709730092145037e-06,
    8.039525710046291e-06,
    7.147788437578129e-06,
    5.158522526471643e-06,
    0.00010011970880441368,
    9.53558901528595e-06,
    0.0012049948563799262,
    1.6588877770118415e-05,
    3.595966336433776e-05,
    9.681348274170887e-06,
    1.9160424926667474e-05,
    1.2161830227341852e-06,
    1.727232870507578e-06,
    4.299986812839052e-06,
    1.3262979337014258e-05,
    6.165052582218777e-06,
    2.0128749383729883e-05,
    2.3350257833953947e-06,
    1.930630241986364e-05,
    0.0018012443324550986,
    4.5344590944296215e-06,
    2.2142817215353716e-06,
    0.0002450325118843466,
    3.260110815972439e-06,
    9.168401788883784e-07,
    5.824126674269792e-06,
    3.837563326669624e-06,
    2.5575316158210626e-06,
    3.1747022148920223e-06,
    2.3617437818757026e-06,
    3.609630948631093e-05,
    9.215612408297602e-06,
    1.4748868125025183e-05,
    5.277248419588432e-06,
    2.3978448098205263e-06,
    2.646331950018066e-06,
    3.2849316085048486e-06,
    8.350278221769258e-06,
    6.095308435760671e-06,
    1.6651914847898297e-05,
    3.92371803172864e-05,
    1.555311791889835e-05,
    8.224559678637888e-06,
    0.004613615106791258,
    1.341473853244679e-05,
    7.509021088480949e-06,
    0.0008436502539552748,
    3.0088751373114064e-05,
    2.5272016500821337e-05,
    5.890773991268361e-06,
    2.360438884352334e-05,
    1.0444240615470335e-05,
    1.5612205970683135e-05,
    9.28065346670337e-05,
    8.607501513324678e-06,
    4.1534003685228527e-05,
    4.636263838619925e-05,
    4.200924377073534e-05,
    4.108679604541976e-06,
    3.6391229514265433e-06,
    1.009377319860505e-05,
    7.623802048328798e-06,
    4.836435437027831e-06,
    0.000456394103821367,
    0.0006979267927818,
    0.00030531815718859434,
    0.00045294544543139637,
    0.000456394103821367,
    0.00014084915164858103,
    0.007669131737202406,
    0.00010398964514024556,
    7.280600402737036e-05,
    0.00013663993740919977,
    0.00018228607950732112,
    0.0003635105094872415,
    0.0002468980383127928,
    0.00010883148206630722,
    0.0009525064961053431,
    0.0005152829689905047,
    0.00017155396926682442,
    0.0011169768404215574,
    0.00011920208635274321,
    0.0003007214399985969,
    1.549424086988438e-05,
    0.0001387285883538425,
    0.00026838164194487035,
    9.749676246428862e-05,
    2.1387968445196748e-05,
    1.5125309573704726e-06,
    2.156271420972189e-06,
    0.0009170593693852425,
    1.36715461849235e-05,
    1.2024245279462775e-06,
    0.00013255664089228958,
    1.1820768577308627e-06,
    2.6363140932517126e-06,
    1.0895173545577563e-06,
    9.099118187805288e-07,
    8.877561299414083e-07,
    2.115769802912837e-06,
    9.530330135021359e-05,
    3.5170048704458168e-06,
    0.00033952368539758027,
    3.911025487468578e-06,
    1.1570371498237364e-05,
    3.5437849419395206e-06,
    1.9686774521687767e-06,
    1.1664906196529046e-06,
    1.6132594282680657e-06,
    1.5980357375156018e-06,
    1.0998966217812267e-06,
    1.8598099131850176e-06,
    1.4100478438194841e-06,
    6.211989784787875e-06,
    0.0005152829689905047,
    2.4437463252979796e-06,
    1.3270311001178925e-06,
    0.0001531049347249791,
    1.5182785091383266e-06,
    3.5256272212791373e-07,
    1.6535213944735005e-06,
    8.074528636825562e-07,
    1.4841194797554635e-06,
    6.566787078554626e-07,
    3.894070687238127e-05,
    2.8985023163841106e-06,
    2.509490286684013e-06,
    1.4343181646836456e-06,
    4.910364168608794e-06,
    2.1077603378216736e-06,
    8.136011047099601e-07,
    2.519026338632102e-06,
    5.607384082395583e-06,
    0.008857968263328075,
    0.0042122299782931805,
    0.012273846194148064,
    0.789665162563324,
    0.0066904048435389996,
    0.0008000252419151366,
    0.0025148559361696243,
    0.0004886376555077732,
    0.0009453090606257319,
    0.0018012443324550986,
    0.003181488486006856,
    0.001299948780797422,
    0.002651990158483386,
    0.0020336597226560116,
    0.0014901157701388001,
    0.0038750481326133013,
    0.0009101297473534942,
    0.0007304228493012488,
    0.0009670663275755942,
    0.0011254814453423023,
    0.0005730112898163497,
    0.00014966042363084853,
    1.6980061445792671e-06,
    1.4673297528133844e-06,
    1.5680193428124767e-06,
    0.0012421143474057317,
    2.1892319637117907e-06,
    1.6379192402382614e-06,
    7.736061525065452e-05,
    1.1933379937545396e-06,
    6.654537401118432e-07,
    6.431230872294691e-07,
    1.1337791647747508e-06,
    1.4869372080283938e-06,
    1.8739693814495695e-06,
    9.571824193699285e-06,
    3.6807630294788396e-06,
    0.0011254814453423023,
    4.449287189345341e-06,
    5.100165253679734e-06,
    1.573976305735414e-06,
    1.2300989737923373e-06,
    1.211579046866973e-06,
    1.3195034398449934e-06,
    3.5170048704458168e-06,
    4.27795441737544e-07,
    9.741961548570544e-07,
    5.693097136827419e-06,
    1.1337791647747508e-06,
    6.829788617324084e-06,
    0.0006323921261355281,
    9.866686013992876e-06,
    3.837563326669624e-06,
    3.410019417060539e-05,
    5.360877253224317e-07,
    2.769547108982806e-06,
    2.103767883454566e-06,
    4.434894265159528e-07,
    3.69474992112373e-06,
    5.453151175061066e-07,
    8.287184755317867e-06,
    2.316102290933486e-05,
    1.1956037724303314e-06,
    0.00022202427498996258,
    2.790635562632815e-06,
    7.59494196245214e-06,
    5.663932256538828e-07,
    1.9575081751099788e-06,
    1.1570371498237364e-05,
    5.310288884174952e-07,
    4.349188202468213e-06,
    1.961224825208774e-06,
    8.260368531409767e-07,
    7.219870781227655e-07,
    2.3261864043888636e-06,
    0.0005773741286247969,
    5.132096134730091e-07,
    3.200640605882654e-07,
    2.231141934316838e-06,
    6.951186151127331e-07,
    4.839109237764205e-07,
    5.890773991268361e-06,
    1.928038500409457e-06,
    3.1626839245291194e-06,
    5.958178007858805e-06,
    1.2231212167534977e-06
   ],
   "p_oracle": [
    0.00025002742768265307,
    4.46088015451096e-05,
    0.0002776079054456204,
    0.002153354696929455,
    3.0440554837696254e-05,
    4.12209628848359e-05,
    0.010080167092382908,
    9.45323881751392e-06,
    6.151765410322696e-05,
    1.8340773749514483e-05,
    6.303364352788776e-05,
    0.000134460351546295,
    0.0003187331312801689,
    0.0001966945274034515,
    0.00026633183006197214,
    0.0005476878141053021,
    0.10250557959079742,
    0.0005888246232643723,
    0.00010699114500312135,
    4.10388020100072e-05,
    9.755772225616965e-06,
    2.2920459741726518e-05,
    1.397377945977496e-05,
    0.00010365874186391011,
    3.376107633812353e-05,
    2.638287332956679e-05,
    8.450755558442324e-06,
    1.3826399481331464e-05,
    3.092841507168487e-05,
    0.0009590017143636942,
    3.7675163184758276e-05,
    2.9886575703130802e-06,
    0.00044685081229545176,
    2.8580673188116634e-06,
    3.415219907765277e-05,
    1.2606312793650432e-06,
    5.314572263159789e-06,
    9.08507627173094e-06,
    7.757460480206646e-06,
    5.504174168891041e-06,
    0.0001035744498949498,
    1.0271732207911555e-05,
    0.0012925089104101062,
    1.815296127460897e-05,
    3.7521804188145325e-05,
    1.0467853826412465e-05,
    2.1318810468073934e-05,
    1.3813976238452597e-06,
    2.0170630250504473e-06,
    4.791125320480205e-06,
    1.5160945622483268e-05,
    7.041288881737273e-06,
    2.1991934772813693e-05,
    2.6158904802286997e-06,
    2.0105630028410815e-05,
    0.0018722949316725135,
    4.904116849502316e-06,
    2.3584395876241615e-06,
    0.0002681392652448267,
    3.5835375911119627e-06,
    9.898735697788652e-07,
    6.462861165346112e-06,
    4.241120677761501e-06,
    2.8101662792323623e-06,
    3.38765858032275e-06,
    2.560000893936376e-06,
    3.559692777344026e-05,
    1.0295524589309935e-05,
    1.5526977222179994e-05,
    5.55905353394337e-06,
    2.6287370928912424e-06,
    2.8178467346151592e-06,
    3.5304301491123624e-06,
    9.378707545693032e-06,
    6.558910627063597e-06,
    1.7312577256234363e-05,
    3.859149001073092e-05,
    1.5665762475691736e-05,
    8.486460501444526e-06,
    0.004536079242825508,
    1.3684801160707138e-05,
    7.760050721117295e-06,
    0.000932063499931246,
    3.2867726986296475e-05,
    2.6262752726324834e-05,
    6.265872343647061e-06,
    2.4607821615063585e-05,
    1.031411829899298e-05,
    1.6193613191717304e-05,
    8.990032802103087e-05,
    8.913102647056803e-06,
    4.179880124866031e-05,
    4.6196688344934955e-05,
    4.2383529944345355e-05,
    4.190259005554253e-06,
    3.84298346034484e-06,
    1.0681268577172887e-05,
    7.898664989625104e-06,
    4.9856180339702405e-06,
    0.0005160845466889441,
    0.0007534350734204054,
    0.00030761174275539815,
    0.0005025957361795008,
    0.00047172774793580174,
    0.0001443958462914452,
    0.008933906443417072,
    0.00011366524995537475,
    7.984249532455578e-05,
    0.0001494999451097101,
    0.00020035890338476747,
    0.0003818984259851277,
    0.0002546251635067165,
    0.00012244832760188729,
    0.000967165979091078,
    0.0005819307989440858,
    0.00018287496641278267,
    0.0011315427254885435,
    0.0001284415484406054,
    0.0003264852857682854,
    1.7358141121803783e-05,
    0.00014977251703385264,
    0.0002880980900954455,
    0.0001065133765223436,
    2.2994790924713016e-05,
    1.6497568822160247e-06,
    2.1944283616903704e-06,
    0.000908317044377327,
    1.4406166883418337e-05,
    1.305515752392239e-06,
    0.00015332236944232136,
    1.3743150475420407e-06,
    2.9563834686996415e-06,
    1.2322205975578981e-06,
    1.0209770380242844e-06,
    9.499912607680017e-07,
    2.4179607862606645e-06,
    9.411927021574229e-05,
    3.81620566258789e-06,
    0.0003593016881495714,
    4.156928753218381e-06,
    1.2415222045092378e-05,
    3.7346069348132005e-06,
    2.224462150479667e-06,
    1.3155066653780523e-06,
    1.8547406170910108e-06,
    1.8464334061718546e-06,
    1.1602907079577562e-06,
    2.0434843008843018e-06,
    1.5312392633859417e-06,
    6.065543402655749e-06,
    0.0005397693603299558,
    2.6460618300916394e-06,
    1.470453639740299e-06,
    0.00017843724344857037,
    1.6811198975119623e-06,
    3.994110215899127e-07,
    1.8138069890483166e-06,
    8.905714139473275e-07,
    1.590703959664097e-06,
    7.238458010760951e-07,
    3.841959551209584e-05,
    3.141479282930959e-06,
    2.7851613140228437e-06,
    1.5261075532180257e-06,
    5.0632124839467e-06,
    2.1783876036352012e-06,
    9.045234037330374e-07,
    2.7719968329620315e-06,
    6.216067049535923e-06,
    0.00926232524216175,
    0.0044129518792033195,
    0.011750814504921436,
    0.7887296080589294,
    0.006786466110497713,
    0.0008130966452881694,
    0.002759390976279974,
    0.0005234149284660816,
    0.000983720296062529,
    0.0018067867495119572,
    0.003228430636227131,
    0.0013381121680140495,
    0.0026090804021805525,
    0.0020524715073406696,
    0.00155772571451962,
    0.003814505413174629,
    0.0009453455568291247,
    0.0007414669962599874,
    0.0010071705328300595,
    0.0012325868010520935,
    0.0006048671202734113,
    0.00015539680316578597,
    1.9223650724597974e-06,
    1.5801366544110351e-06,
    1.6460982124044676e-06,
    0.0013403076445683837,
    2.2976716991252033e-06,
    1.8359454543315223e-06,
    9.20241727726534e-05,
    1.333405748482619e-06,
    7.37509481041343e-07,
    7.530991297244327e-07,
    1.282306584471371e-06,
    1.614519987924723e-06,
    2.0336372017482063e-06,
    1.0205317266809288e-05,
    4.1190914998878725e-06,
    0.0011807188857346773,
    4.89960984850768e-06,
    5.530182079382939e-06,
    1.7047153733074083e-06,
    1.3161767355995835e-06,
    1.3940655207989039e-06,
    1.5497056438107393e-06,
    3.8067626064730575e-06,
    5.0939985385412e-07,
    1.0830405017259181e-06,
    6.424417733796872e-06,
    1.262598516404978e-06,
    7.240449576784158e-06,
    0.0006854813545942307,
    1.0710738024499733e-05,
    4.121913207200123e-06,
    4.0812898078002036e-05,
    6.273821213653719e-07,
    3.200123273927602e-06,
    2.461954636601149e-06,
    5.171972929929325e-07,
    3.900078354490688e-06,
    5.871208372809633e-07,
    8.984105079434812e-06,
    2.3507813239120878e-05,
    1.3550717312682536e-06,
    0.00022512504074256867,
    3.126439878542442e-06,
    8.371463991352357e-06,
    6.070031304261647e-07,
    2.1609405393974157e-06,
    1.2610324120032601e-05,
    6.137665309324802e-07,
    4.585451733873924e-06,
    2.1809028112329543e-06,
    9.200776389661769e-07,
    7.936535553199064e-07,
    2.4281953301397152e-06,
    0.0006229236023500562,
    5.834775720359175e-07,
    3.6477001685852883e-07,
    2.591647671579267e-06,
    7.93853416780621e-07,
    5.387703367887298e-07,
    5.918919214309426e-06,
    2.2097124201536644e-06,
    3.4735091958282283e-06,
    6.435025625251001e-06,
    1.3190820027375594e-06
   ],
   "oracle_margin": 0.6862240433692932,
   "max_abs_delta_p": 0.003299310803413391,
   "mean_abs_delta_p": 3.885284526021161e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 17.880274042021483
  },
  {
   "id": "r13-status",
   "tokens": 52,
   "nopts": 3,
   "letter_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 33,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.0015108715742826462,
    0.9976529479026794,
    0.0008361565414816141
   ],
   "p_oracle": [
    0.001543335267342627,
    0.9976155757904053,
    0.0008411174057982862
   ],
   "oracle_margin": 0.9960722327232361,
   "max_abs_delta_p": 3.737211227416992e-05,
   "mean_abs_delta_p": 2.4932223216940958e-05,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.4892234168946743
  },
  {
   "id": "r13-restart",
   "tokens": 48,
   "nopts": 2,
   "letter_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "full_vocab_argmax_id": 32,
   "full_vocab_argmax_is_label": true,
   "p": [
    0.9895533323287964,
    0.010446663945913315
   ],
   "p_oracle": [
    0.9900590777397156,
    0.00994098000228405
   ],
   "oracle_margin": 0.9801180958747864,
   "max_abs_delta_p": 0.0005057454109191895,
   "mean_abs_delta_p": 0.0005057146772742271,
   "finite": true,
   "nonconstant": true,
   "seconds": 0.41807829204481095
  }
 ],
 "reset_check": {
  "row": "r01-route",
  "identical": true,
  "max_abs_diff": 0.0
 },
 "summary": {
  "rows": 44,
  "expected_rows": 44,
  "argmax_agreement": 44,
  "full_vocab_argmax_is_label": 44,
  "max_abs_delta_p": 0.008350849151611328,
  "mean_of_row_mean_abs_delta_p": 0.0006699172689245611,
  "finite_all": true,
  "reset_identical": true,
  "bar": {
   "max_abs_delta_p": 0.02,
   "mean_of_row_mean_abs_delta_p": 0.002
  }
 },
 "result": "PASS",
 "generated_at": "2026-09-21T07:43:17+00:00"
}