{
 "schema": "coreai-decider-readout-gate/1",
 "head": "slot",
 "mode": "int8lin",
 "bundle": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/openthai_systemone_decode_int8lin",
 "fixtures": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/models/openthai-systemone/fixtures-openthai-systemone.json",
 "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
 "max_ctx": 4096,
 "runtime": "coreai Python runtime, AOT h16c GPU asset, SpecializationOptions.default()",
 "source": {
  "hf_id": "iapp/OpenThai-SystemOne",
  "revision": "f3709948b5e3cc9606a57e74ba62b7a639d17dd3",
  "oracle": "author's modeling.py slot_logits, fp32, CPU",
  "transformers": "5.17.0",
  "torch": "2.9.0",
  "client_revision": "5d04bcc",
  "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/snapshots/f3709948b5e3cc9606a57e74ba62b7a639d17dd3"
 },
 "temperature_by_type": {
  "choice": 1.058534,
  "score": 1.043141,
  "noul": 1.006767
 },
 "n_slots": 256,
 "abstain_slot": 255,
 "answer_token_id": 248082,
 "environment": {
  "python": "3.11.13",
  "platform": "darwin",
  "DEVELOPER_DIR": "/Applications/Xcode-27.0.0-RC.app/Contents/Developer"
 },
 "started_at": "2026-09-22T19:27:36+00:00",
 "timing": "contended; correctness only, no performance claim",
 "rows": [
  {
   "id": "r01-route",
   "request_id": "r01",
   "question_id": "route",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 113,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    0.923828125,
    -4.08984375,
    -1.787109375,
    -3.15234375,
    -3.03515625,
    -3.1484375,
    -3.1015625,
    -3.046875,
    -4.265625,
    -2.974609375,
    -3.037109375,
    -2.724609375,
    -4.38671875,
    -3.40234375,
    -3.67578125,
    -3.583984375,
    -3.9765625,
    -4.8984375,
    -3.580078125,
    -3.421875,
    -3.69140625,
    -2.41015625,
    -5.51171875,
    -5.21875,
    -3.9375,
    -4.7890625,
    -3.427734375,
    -4.73828125,
    -4.14453125,
    -5.67578125,
    -4.3125,
    -4.5390625,
    -5.82421875,
    -3.943359375,
    -3.287109375,
    -4.94140625,
    -5.75,
    -6.21484375,
    -4.6875,
    -3.01953125,
    -5.88671875,
    -4.8046875,
    -5.15234375,
    -5.96484375,
    -3.826171875,
    -4.96484375,
    -4.35546875,
    -4.94921875,
    -5.39453125,
    -4.703125,
    -5.26953125,
    -4.6328125,
    -3.884765625,
    -5.47265625,
    -3.40625,
    -3.783203125,
    -3.955078125,
    -4.4453125,
    -4.58203125,
    -5.77734375,
    -7.453125,
    -7.21484375,
    -8.140625,
    -5.8203125,
    -3.728515625,
    -7.51171875,
    -8.03125,
    -2.712890625,
    -7.87109375,
    -4.35546875,
    -8.0078125,
    -5.390625,
    -2.765625,
    -5.41796875,
    -3.7734375,
    -6.3828125,
    -4.125,
    -7.0390625,
    -4.0390625,
    -6.0703125,
    -4.66015625,
    -4.265625,
    -3.19140625,
    -4.75,
    -3.564453125,
    -4.89453125,
    -2.65625,
    -4.12109375,
    -3.328125,
    -4.48828125,
    -4.02734375,
    -4.14453125,
    -4.31640625,
    -3.615234375,
    -4.78515625,
    -4.1171875,
    -5.1015625,
    -3.16015625,
    -2.66796875,
    -4.76171875,
    -6.18359375,
    -4.96875,
    -3.4140625,
    -4.359375,
    -4.7890625,
    -5.94921875,
    -3.48046875,
    -5.23828125,
    -4.140625,
    -6.0234375,
    -4.1796875,
    -4.98046875,
    -5.04296875,
    -5.79296875,
    -4.703125,
    -3.51171875,
    -4.7890625,
    -6.01953125,
    -4.265625,
    -3.701171875,
    -3.140625,
    -4.2890625,
    -2.89453125,
    -3.689453125,
    -2.212890625,
    -5.1171875,
    -5.84375,
    -2.671875,
    -2.82421875,
    -5.80078125,
    -5.6875,
    -3.255859375,
    -5.8203125,
    -6.65625,
    -5.3671875,
    -4.9609375,
    -3.77734375,
    -4.33984375,
    -6.3671875,
    -2.70703125,
    -3.63671875,
    -4.328125,
    -3.044921875,
    -2.8046875,
    -4.25390625,
    -4.93359375,
    -5.8984375,
    -5.80078125,
    -5.22265625,
    -4.51953125,
    -4.5546875,
    -2.87109375,
    -1.0556640625,
    -3.166015625,
    -1.939453125,
    -1.4638671875,
    -1.576171875,
    -1.51953125,
    -2.634765625,
    -0.87158203125,
    -1.861328125,
    -1.751953125,
    -1.4228515625,
    -1.3427734375,
    -2.6953125,
    -1.84765625,
    -2.994140625,
    -2.28515625,
    -3.892578125,
    -2.1171875,
    -2.21875,
    -2.548828125,
    -1.87890625,
    -2.92578125,
    -0.955078125,
    -1.955078125,
    -3.685546875,
    -3.427734375,
    -3.251953125,
    -2.583984375,
    -3.9296875,
    -2.513671875,
    -3.091796875,
    -2.76953125,
    -3.40625,
    -2.51953125,
    -2.005859375,
    -2.0625,
    -2.54296875,
    -2.66796875,
    -1.275390625,
    -2.015625,
    -2.771484375,
    -1.85546875,
    -2.064453125,
    -2.109375,
    -2.08203125,
    -2.03515625,
    -2.125,
    -1.1337890625,
    -1.404296875,
    -2.62890625,
    -1.26953125,
    -2.28125,
    -2.572265625,
    -1.7861328125,
    -2.2109375,
    -2.498046875,
    0.06658935546875,
    -2.31640625,
    -1.7529296875,
    -1.2412109375,
    -0.11883544921875,
    -1.1279296875,
    -3.3125,
    -1.12890625,
    -2.02734375,
    -2.14453125,
    -1.5546875,
    -2.111328125,
    -1.2255859375,
    -1.3544921875,
    -1.0224609375,
    -0.8388671875,
    -0.1019287109375,
    -0.44970703125,
    -0.771484375,
    0.9013671875,
    0.74267578125,
    -0.39111328125,
    -0.036651611328125,
    -0.456298828125,
    0.11053466796875,
    -0.9267578125,
    0.09326171875,
    1.0791015625,
    0.89404296875,
    -0.0767822265625,
    -0.09173583984375,
    -0.7197265625,
    0.303466796875,
    0.54296875,
    -0.71826171875,
    0.053619384765625,
    -0.140380859375,
    0.416748046875,
    0.7802734375,
    -0.76318359375,
    -0.1309814453125,
    2.05859375,
    -0.72607421875,
    -0.27685546875,
    -0.50244140625,
    0.0005345344543457031,
    -0.30810546875,
    -2.701171875
   ],
   "raw256_argmax": 249,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.8940073847770691,
    0.007840349338948727,
    0.06903982162475586,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.029112448915839195
   ],
   "p": [
    0.920814573764801,
    0.008075444959104061,
    0.07111001014709473
   ],
   "p_oracle": [
    0.9192458987236023,
    0.008343691937625408,
    0.072410449385643
   ],
   "abstain": 0.029112448915839195,
   "abstain_oracle": 0.02907981537282467,
   "abs_delta_abstain": 3.263354301452637e-05,
   "oracle_margin": 0.8468354344367981,
   "max_abs_delta_p": 0.0015686750411987305,
   "mean_abs_delta_p": 0.001045787086089452,
   "max_abs_delta_raw_logits": 0.030035972595214844,
   "mean_abs_delta_raw_logits": 0.006498387403553352,
   "logits_min": -8.140625,
   "logits_max": 2.05859375,
   "raw_logits_min": -8.140625,
   "raw_logits_max": 2.05859375,
   "raw_logits_absmax": 8.140625,
   "oracle_logits_min": -8.13015365600586,
   "oracle_logits_max": 2.0633702278137207,
   "oracle_raw_logits_absmax": 8.13015365600586,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 2.1197234999854118,
   "seconds": 2.1197234999854118,
   "full_vocab_argmax_id": 249,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r01-condition",
   "request_id": "r01",
   "question_id": "condition",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 87,
   "nopts": 4,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -3.216796875,
    3.1484375,
    -2.623046875,
    -0.7939453125,
    -2.3125,
    -3.03515625,
    -2.84375,
    -3.623046875,
    -4.11328125,
    -3.08984375,
    -3.048828125,
    -3.98828125,
    -5.421875,
    -3.08203125,
    -4.1796875,
    -4.73828125,
    -4.15234375,
    -4.44921875,
    -2.9609375,
    -2.994140625,
    -3.349609375,
    -1.6669921875,
    -5.265625,
    -6.65625,
    -4.1015625,
    -5.39453125,
    -2.58984375,
    -5.83203125,
    -4.40625,
    -6.66015625,
    -4.671875,
    -4.8984375,
    -6.25390625,
    -4.625,
    -2.9453125,
    -5.48046875,
    -5.1640625,
    -5.61328125,
    -4.00390625,
    -2.94140625,
    -7.265625,
    -4.25390625,
    -4.92578125,
    -5.90625,
    -4.43359375,
    -5.0859375,
    -4.9140625,
    -4.8046875,
    -5.296875,
    -4.609375,
    -5.6953125,
    -4.859375,
    -2.453125,
    -5.0546875,
    -2.44140625,
    -3.08984375,
    -3.763671875,
    -4.50390625,
    -4.7265625,
    -5.73828125,
    -7.65234375,
    -6.453125,
    -7.59375,
    -6.56640625,
    -3.755859375,
    -7.5546875,
    -7.6015625,
    -2.162109375,
    -7.75390625,
    -4.2578125,
    -8.1796875,
    -5.3828125,
    -2.560546875,
    -6.3125,
    -3.861328125,
    -6.375,
    -4.40625,
    -6.6796875,
    -4.14453125,
    -5.57421875,
    -4.18359375,
    -4.05078125,
    -2.46875,
    -3.96484375,
    -3.41015625,
    -4.02734375,
    -2.720703125,
    -3.642578125,
    -3.0546875,
    -4.8828125,
    -3.5859375,
    -4.125,
    -4.2109375,
    -4.34375,
    -4.71875,
    -4.953125,
    -5.2109375,
    -4.11328125,
    -4.1484375,
    -3.923828125,
    -6.015625,
    -4.89453125,
    -3.486328125,
    -4.37109375,
    -4.38671875,
    -6.21875,
    -4.35546875,
    -5.66015625,
    -4.30078125,
    -5.328125,
    -4.45703125,
    -4.8046875,
    -5.20703125,
    -6.61328125,
    -4.9296875,
    -4.30859375,
    -4.6328125,
    -6.125,
    -4.6015625,
    -3.3515625,
    -3.181640625,
    -4.12109375,
    -3.3125,
    -3.8359375,
    -2.54296875,
    -5.96484375,
    -5.79296875,
    -3.37890625,
    -3.275390625,
    -5.52734375,
    -5.5234375,
    -3.798828125,
    -5.75390625,
    -6.09375,
    -5.09765625,
    -4.75390625,
    -4.0625,
    -5.04296875,
    -5.890625,
    -2.9609375,
    -3.74609375,
    -4.1953125,
    -3.794921875,
    -3.52734375,
    -4.80859375,
    -5.29296875,
    -6.30078125,
    -5.26953125,
    -4.8828125,
    -4.40234375,
    -4.78125,
    -3.154296875,
    -1.33984375,
    -3.67578125,
    -2.650390625,
    -1.80859375,
    -2.033203125,
    -2.15625,
    -3.4296875,
    -0.783203125,
    -2.400390625,
    -2.255859375,
    -1.7568359375,
    -1.3427734375,
    -2.7734375,
    -2.40625,
    -2.875,
    -1.73828125,
    -3.375,
    -2.19140625,
    -2.095703125,
    -2.59765625,
    -2.525390625,
    -3.12109375,
    -1.1640625,
    -2.015625,
    -3.255859375,
    -3.427734375,
    -3.095703125,
    -3.650390625,
    -3.55859375,
    -3.2890625,
    -2.703125,
    -2.015625,
    -2.634765625,
    -1.5234375,
    -1.40234375,
    -1.7890625,
    -1.9287109375,
    -1.900390625,
    -1.2119140625,
    -1.69140625,
    -3.013671875,
    -1.5810546875,
    -1.34765625,
    -2.13671875,
    -1.9296875,
    -1.8935546875,
    -1.7353515625,
    -1.3134765625,
    -1.3720703125,
    -1.96484375,
    -1.916015625,
    -1.79296875,
    -2.365234375,
    -1.119140625,
    -2.673828125,
    -2.734375,
    -0.387451171875,
    -2.296875,
    -0.79150390625,
    -1.3408203125,
    -0.83544921875,
    -1.5068359375,
    -2.4453125,
    -1.482421875,
    -1.869140625,
    -2.44140625,
    -1.8583984375,
    -2.04296875,
    -1.3974609375,
    -1.734375,
    -1.57421875,
    -1.4189453125,
    -0.203369140625,
    0.260009765625,
    -0.341064453125,
    0.7490234375,
    0.94677734375,
    -0.1063232421875,
    -1.0771484375,
    -0.491943359375,
    -0.10015869140625,
    -0.9150390625,
    0.85498046875,
    1.126953125,
    1.546875,
    -0.6591796875,
    0.07122802734375,
    -0.77490234375,
    0.31884765625,
    0.69873046875,
    -0.779296875,
    -0.08642578125,
    0.142822265625,
    0.5498046875,
    0.2437744140625,
    -0.330322265625,
    -0.66650390625,
    1.484375,
    0.04486083984375,
    -0.736328125,
    -0.4111328125,
    0.556640625,
    -0.56591796875,
    -4.59765625
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0023713556583970785,
    0.9694393277168274,
    0.004155279137194157,
    0.023390669375658035,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0006433671806007624
   ],
   "p": [
    0.002372882328927517,
    0.9700634479522705,
    0.00415795436128974,
    0.023405728861689568
   ],
   "p_oracle": [
    0.0023138143587857485,
    0.9696792364120483,
    0.004179855342954397,
    0.023827005177736282
   ],
   "abstain": 0.0006433671806007624,
   "abstain_oracle": 0.000638672907371074,
   "abs_delta_abstain": 4.694273229688406e-06,
   "oracle_margin": 0.9458522200584412,
   "max_abs_delta_p": 0.0004212763160467148,
   "mean_abs_delta_p": 0.0002216142020188272,
   "max_abs_delta_raw_logits": 0.030066490173339844,
   "mean_abs_delta_raw_logits": 0.005172437115106732,
   "logits_min": -8.1796875,
   "logits_max": 3.1484375,
   "raw_logits_min": -8.1796875,
   "raw_logits_max": 3.1484375,
   "raw_logits_absmax": 8.1796875,
   "oracle_logits_min": -8.181787490844727,
   "oracle_logits_max": 3.1483166217803955,
   "oracle_raw_logits_absmax": 8.181787490844727,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.48780895792879164,
   "seconds": 0.48780895792879164,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r01-refund",
   "request_id": "r01",
   "question_id": "refund",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 87,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    3.19921875,
    -2.443359375,
    -2.138671875,
    -2.775390625,
    -2.443359375,
    -2.693359375,
    -2.953125,
    -3.892578125,
    -3.23046875,
    -2.3046875,
    -2.75,
    -3.359375,
    -4.41796875,
    -2.77734375,
    -4.078125,
    -4.4453125,
    -4.5390625,
    -4.65625,
    -2.681640625,
    -3.455078125,
    -3.35546875,
    -1.5458984375,
    -4.90234375,
    -5.8984375,
    -3.84375,
    -4.38671875,
    -2.978515625,
    -5.06640625,
    -4.13671875,
    -6.59375,
    -4.109375,
    -5.28125,
    -5.9765625,
    -4.328125,
    -2.978515625,
    -4.953125,
    -5.16796875,
    -6.29296875,
    -4.453125,
    -3.568359375,
    -6.4453125,
    -4.35546875,
    -5.23828125,
    -6.1171875,
    -4.1640625,
    -4.81640625,
    -4.1015625,
    -4.86328125,
    -5.12890625,
    -4.1015625,
    -5.3828125,
    -4.23046875,
    -3.349609375,
    -5.4921875,
    -3.205078125,
    -3.3828125,
    -4.15625,
    -4.89453125,
    -4.74609375,
    -6.65234375,
    -7.46484375,
    -7.08203125,
    -8.0859375,
    -5.90234375,
    -3.876953125,
    -7.40625,
    -7.734375,
    -2.556640625,
    -7.5234375,
    -4.25390625,
    -8.078125,
    -5.3359375,
    -3.3671875,
    -5.5859375,
    -3.69140625,
    -6.43359375,
    -4.28515625,
    -7.18359375,
    -3.99609375,
    -6.14453125,
    -4.640625,
    -4.1796875,
    -3.05078125,
    -4.6015625,
    -3.654296875,
    -4.58984375,
    -2.587890625,
    -4.203125,
    -3.228515625,
    -4.859375,
    -3.751953125,
    -4.31640625,
    -4.26953125,
    -3.73046875,
    -4.62890625,
    -4.61328125,
    -5.0234375,
    -3.537109375,
    -3.142578125,
    -4.25,
    -5.76171875,
    -4.96484375,
    -3.396484375,
    -4.0546875,
    -4.5546875,
    -5.8203125,
    -4.11328125,
    -5.3125,
    -4.47265625,
    -6.26953125,
    -4.62109375,
    -4.875,
    -5.23828125,
    -6.05859375,
    -5.06640625,
    -3.916015625,
    -4.6640625,
    -5.9765625,
    -3.81640625,
    -4.0390625,
    -3.55859375,
    -4.484375,
    -3.23046875,
    -3.751953125,
    -2.451171875,
    -5.39453125,
    -5.796875,
    -2.86328125,
    -2.984375,
    -5.8046875,
    -6.203125,
    -2.859375,
    -5.9140625,
    -6.5,
    -5.03125,
    -4.08203125,
    -3.888671875,
    -4.87109375,
    -6.21875,
    -2.966796875,
    -3.421875,
    -4.33203125,
    -3.50390625,
    -2.572265625,
    -5.2109375,
    -5.15234375,
    -6.0078125,
    -5.7109375,
    -5.359375,
    -4.74609375,
    -4.6796875,
    -2.7578125,
    -1.177734375,
    -3.51953125,
    -3.048828125,
    -1.6171875,
    -1.8271484375,
    -1.837890625,
    -2.8828125,
    -0.48828125,
    -1.98046875,
    -2.25,
    -1.6865234375,
    -0.95263671875,
    -2.970703125,
    -1.97265625,
    -2.853515625,
    -2.61328125,
    -3.943359375,
    -2.015625,
    -2.244140625,
    -2.556640625,
    -2.244140625,
    -3.091796875,
    -0.8525390625,
    -2.216796875,
    -3.73828125,
    -3.466796875,
    -2.861328125,
    -2.439453125,
    -3.365234375,
    -2.564453125,
    -3.244140625,
    -2.11328125,
    -2.568359375,
    -1.8251953125,
    -1.4755859375,
    -1.53125,
    -2.212890625,
    -2.9140625,
    -0.7705078125,
    -1.837890625,
    -2.53515625,
    -1.5419921875,
    -1.830078125,
    -1.7958984375,
    -2.197265625,
    -1.9765625,
    -2.224609375,
    -1.36328125,
    -1.287109375,
    -2.05078125,
    -1.73828125,
    -1.98046875,
    -2.5078125,
    -1.16796875,
    -2.203125,
    -2.5546875,
    0.218505859375,
    -2.078125,
    -1.5107421875,
    -1.1015625,
    -0.0233154296875,
    -0.857421875,
    -2.66015625,
    -0.92578125,
    -1.6865234375,
    -1.982421875,
    -1.2529296875,
    -1.9072265625,
    -0.958984375,
    -1.5126953125,
    -0.640625,
    -1.087890625,
    -0.353515625,
    -0.2193603515625,
    -0.55078125,
    0.9267578125,
    0.8388671875,
    -0.31640625,
    -0.418212890625,
    -0.7412109375,
    0.492919921875,
    -0.70654296875,
    0.379150390625,
    0.353271484375,
    1.205078125,
    -0.293701171875,
    0.2218017578125,
    -0.89111328125,
    0.2978515625,
    0.76806640625,
    -0.890625,
    -0.455810546875,
    -0.1253662109375,
    0.96142578125,
    0.54248046875,
    -0.732421875,
    -0.7392578125,
    2.224609375,
    -0.011322021484375,
    0.0191497802734375,
    -0.26708984375,
    0.363037109375,
    -0.373291015625,
    -4.765625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9959690570831299,
    0.0036658700555562973,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003650977450888604
   ],
   "p": [
    0.9963328242301941,
    0.003667208831757307
   ],
   "p_oracle": [
    0.9963294863700867,
    0.0036705152597278357
   ],
   "abstain": 0.0003650977450888604,
   "abstain_oracle": 0.00036291085416451097,
   "abs_delta_abstain": 2.1868909243494272e-06,
   "oracle_margin": 0.9926589727401733,
   "max_abs_delta_p": 3.337860107421875e-06,
   "mean_abs_delta_p": 3.322144038975239e-06,
   "max_abs_delta_raw_logits": 0.031358957290649414,
   "mean_abs_delta_raw_logits": 0.0061593796126544476,
   "logits_min": -8.0859375,
   "logits_max": 3.19921875,
   "raw_logits_min": -8.0859375,
   "raw_logits_max": 3.19921875,
   "raw_logits_absmax": 8.0859375,
   "oracle_logits_min": -8.083602905273438,
   "oracle_logits_max": 3.207350730895996,
   "oracle_raw_logits_absmax": 8.083602905273438,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.48581237508915365,
   "seconds": 0.48581237508915365,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r02-action",
   "request_id": "r02",
   "question_id": "action",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 108,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    3.55859375,
    -2.404296875,
    -1.703125,
    -2.626953125,
    -2.115234375,
    -2.576171875,
    -2.29296875,
    -3.55078125,
    -3.072265625,
    -2.203125,
    -2.625,
    -3.447265625,
    -4.4453125,
    -2.498046875,
    -3.94140625,
    -3.955078125,
    -3.98828125,
    -4.7734375,
    -2.71875,
    -3.099609375,
    -3.189453125,
    -1.8447265625,
    -5.19140625,
    -6.09375,
    -3.7421875,
    -4.19921875,
    -2.78125,
    -4.6484375,
    -3.9609375,
    -6.19921875,
    -3.904296875,
    -4.8046875,
    -5.70703125,
    -4.171875,
    -3.005859375,
    -4.5,
    -5.0390625,
    -6.01171875,
    -4.1953125,
    -3.259765625,
    -6.2734375,
    -4.30078125,
    -4.953125,
    -6.1953125,
    -3.853515625,
    -4.609375,
    -4.09765625,
    -4.5546875,
    -5.1015625,
    -4.22265625,
    -5.16796875,
    -4.0703125,
    -3.279296875,
    -5.44921875,
    -3.248046875,
    -3.41015625,
    -4.15234375,
    -4.94140625,
    -4.640625,
    -6.53515625,
    -7.26953125,
    -6.91015625,
    -8.0234375,
    -5.69921875,
    -3.8671875,
    -7.17578125,
    -7.5546875,
    -2.544921875,
    -7.3203125,
    -4.0703125,
    -7.8671875,
    -5.22265625,
    -3.169921875,
    -5.31640625,
    -3.5,
    -5.9140625,
    -4.10546875,
    -6.88671875,
    -3.794921875,
    -5.8984375,
    -4.6328125,
    -4.078125,
    -2.927734375,
    -4.4609375,
    -3.34375,
    -4.40234375,
    -2.380859375,
    -4.1484375,
    -3.01953125,
    -4.60546875,
    -3.80859375,
    -4.0859375,
    -4.3359375,
    -3.556640625,
    -4.4609375,
    -4.42578125,
    -4.890625,
    -3.427734375,
    -2.921875,
    -4.203125,
    -5.44921875,
    -4.72265625,
    -3.21484375,
    -3.83984375,
    -4.5078125,
    -5.50390625,
    -3.654296875,
    -5.03125,
    -4.27734375,
    -5.94921875,
    -4.3828125,
    -4.7109375,
    -5.08203125,
    -5.95703125,
    -4.8203125,
    -3.69921875,
    -4.5234375,
    -5.69140625,
    -3.6640625,
    -3.796875,
    -3.255859375,
    -4.41796875,
    -3.03515625,
    -3.69140625,
    -2.291015625,
    -5.109375,
    -5.7421875,
    -2.423828125,
    -2.818359375,
    -5.46875,
    -5.8671875,
    -2.59375,
    -5.66796875,
    -6.26171875,
    -4.9453125,
    -3.9609375,
    -3.62890625,
    -4.73828125,
    -6.0546875,
    -2.765625,
    -3.5,
    -4.203125,
    -3.2421875,
    -2.298828125,
    -5.0078125,
    -5.10546875,
    -5.7734375,
    -5.390625,
    -5.23828125,
    -4.58203125,
    -4.546875,
    -2.796875,
    -0.9501953125,
    -3.44140625,
    -2.75390625,
    -1.5966796875,
    -1.7275390625,
    -1.9228515625,
    -2.83984375,
    -0.2069091796875,
    -1.6181640625,
    -2.060546875,
    -1.708984375,
    -0.9921875,
    -2.955078125,
    -1.6669921875,
    -2.611328125,
    -2.337890625,
    -3.517578125,
    -1.830078125,
    -2.1953125,
    -2.396484375,
    -2.23828125,
    -2.990234375,
    -0.57373046875,
    -1.8818359375,
    -3.529296875,
    -3.296875,
    -2.85546875,
    -2.076171875,
    -3.025390625,
    -2.478515625,
    -2.93359375,
    -2.04296875,
    -2.427734375,
    -1.9443359375,
    -1.34765625,
    -1.3955078125,
    -2.146484375,
    -2.71484375,
    -0.51171875,
    -1.6240234375,
    -2.267578125,
    -1.7373046875,
    -1.9384765625,
    -1.8359375,
    -2.001953125,
    -1.7119140625,
    -2.07421875,
    -1.10546875,
    -1.1044921875,
    -1.8173828125,
    -1.453125,
    -2.0078125,
    -2.44140625,
    -1.1328125,
    -2.046875,
    -2.458984375,
    0.281494140625,
    -1.970703125,
    -1.4111328125,
    -0.927734375,
    0.08404541015625,
    -0.76953125,
    -2.71875,
    -0.703125,
    -1.56640625,
    -1.8154296875,
    -0.98486328125,
    -1.7373046875,
    -1.041015625,
    -1.83203125,
    -0.71533203125,
    -1.1435546875,
    -0.3271484375,
    -0.1881103515625,
    -0.38916015625,
    1.1484375,
    0.736328125,
    -0.1904296875,
    -0.45703125,
    -0.6943359375,
    0.724609375,
    -0.73388671875,
    0.2381591796875,
    0.56201171875,
    1.134765625,
    -0.1461181640625,
    0.12396240234375,
    -0.7158203125,
    0.283203125,
    0.748046875,
    -0.82958984375,
    -0.470703125,
    -0.1702880859375,
    0.83935546875,
    0.7431640625,
    -0.60302734375,
    -0.876953125,
    2.283203125,
    -0.10479736328125,
    -0.11431884765625,
    -0.2325439453125,
    0.1849365234375,
    -0.405029296875,
    -4.953125
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9892790913581848,
    0.0035388998221606016,
    0.006863484159111977,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00031852012034505606
   ],
   "p": [
    0.9895942807197571,
    0.0035400274209678173,
    0.006865670904517174
   ],
   "p_oracle": [
    0.989686131477356,
    0.0035125669091939926,
    0.006801280193030834
   ],
   "abstain": 0.00031852012034505606,
   "abstain_oracle": 0.00031577961635775864,
   "abs_delta_abstain": 2.7405039872974157e-06,
   "oracle_margin": 0.9828848242759705,
   "max_abs_delta_p": 9.185075759887695e-05,
   "mean_abs_delta_p": 6.12339936196804e-05,
   "max_abs_delta_raw_logits": 0.021271467208862305,
   "mean_abs_delta_raw_logits": 0.005188052251469344,
   "logits_min": -8.0234375,
   "logits_max": 3.55859375,
   "raw_logits_min": -8.0234375,
   "raw_logits_max": 3.55859375,
   "raw_logits_absmax": 8.0234375,
   "oracle_logits_min": -8.02131175994873,
   "oracle_logits_max": 3.5675013065338135,
   "oracle_raw_logits_absmax": 8.02131175994873,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.6038089579669759,
   "seconds": 0.6038089579669759,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r02-lamp",
   "request_id": "r02",
   "question_id": "lamp",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 79,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.259765625,
    -2.92578125,
    3.16796875,
    -1.5966796875,
    -2.123046875,
    -2.37109375,
    -2.369140625,
    -2.810546875,
    -3.01953125,
    -1.8017578125,
    -2.8359375,
    -2.626953125,
    -4.1796875,
    -2.548828125,
    -3.5234375,
    -4.1796875,
    -3.76171875,
    -4.4609375,
    -2.556640625,
    -2.50390625,
    -4.0546875,
    -1.396484375,
    -5.34765625,
    -5.44140625,
    -3.26171875,
    -4.359375,
    -3.302734375,
    -3.841796875,
    -3.693359375,
    -5.79296875,
    -4.35546875,
    -4.94921875,
    -4.94921875,
    -4.12109375,
    -2.740234375,
    -4.8203125,
    -5.27734375,
    -6.546875,
    -4.66015625,
    -3.345703125,
    -5.93359375,
    -3.966796875,
    -4.78125,
    -6.1796875,
    -4.046875,
    -4.76953125,
    -4.30078125,
    -4.34765625,
    -4.86328125,
    -4.8046875,
    -5.17578125,
    -4.8046875,
    -3.05859375,
    -5.03515625,
    -3.009765625,
    -3.5625,
    -4.03125,
    -5.23046875,
    -5.1171875,
    -6.1328125,
    -7.32421875,
    -6.70703125,
    -7.51171875,
    -5.8828125,
    -3.373046875,
    -7.09375,
    -7.375,
    -2.83984375,
    -7.6171875,
    -4.49609375,
    -7.8515625,
    -5.1640625,
    -2.6953125,
    -5.20703125,
    -3.021484375,
    -6.125,
    -3.890625,
    -6.546875,
    -4.140625,
    -5.51171875,
    -4.54296875,
    -3.783203125,
    -2.548828125,
    -4.5625,
    -3.27734375,
    -4.59765625,
    -2.576171875,
    -4.15234375,
    -3.08203125,
    -4.3203125,
    -3.419921875,
    -3.74609375,
    -4.2265625,
    -3.125,
    -4.83203125,
    -4.578125,
    -4.95703125,
    -3.306640625,
    -3.07421875,
    -4.49609375,
    -5.48828125,
    -4.4375,
    -3.583984375,
    -3.603515625,
    -4.33984375,
    -5.35546875,
    -3.134765625,
    -5.125,
    -4.04296875,
    -5.78125,
    -3.890625,
    -4.48046875,
    -4.9296875,
    -5.55859375,
    -4.3671875,
    -3.544921875,
    -4.32421875,
    -5.63671875,
    -3.765625,
    -3.126953125,
    -3.494140625,
    -4.30078125,
    -2.892578125,
    -3.173828125,
    -2.447265625,
    -5.07421875,
    -5.203125,
    -2.947265625,
    -2.654296875,
    -5.359375,
    -5.3203125,
    -2.435546875,
    -5.5078125,
    -6.1796875,
    -5.4453125,
    -4.7734375,
    -3.64453125,
    -4.3046875,
    -5.6328125,
    -3.0,
    -2.96484375,
    -3.978515625,
    -3.46484375,
    -2.68359375,
    -4.05078125,
    -4.70703125,
    -5.1875,
    -5.0625,
    -5.03125,
    -4.2734375,
    -4.51171875,
    -2.779296875,
    -0.55712890625,
    -3.4921875,
    -2.25390625,
    -1.169921875,
    -1.3798828125,
    -1.681640625,
    -2.75,
    -0.1678466796875,
    -1.6796875,
    -2.005859375,
    -1.203125,
    -1.2490234375,
    -3.1640625,
    -2.15625,
    -3.001953125,
    -1.92578125,
    -3.583984375,
    -1.7607421875,
    -2.083984375,
    -1.96875,
    -1.783203125,
    -2.5,
    -0.515625,
    -1.572265625,
    -2.927734375,
    -3.146484375,
    -2.509765625,
    -2.6171875,
    -2.763671875,
    -2.275390625,
    -2.890625,
    -2.083984375,
    -2.912109375,
    -1.642578125,
    -2.19140625,
    -1.28515625,
    -1.4453125,
    -2.548828125,
    -0.61181640625,
    -1.5087890625,
    -2.423828125,
    -1.53125,
    -1.9111328125,
    -2.3125,
    -1.64453125,
    -1.48828125,
    -1.7001953125,
    -1.0947265625,
    -0.833984375,
    -1.9345703125,
    -1.681640625,
    -1.7685546875,
    -2.525390625,
    -0.76416015625,
    -2.14453125,
    -2.32421875,
    -0.0377197265625,
    -1.76171875,
    -0.85302734375,
    -0.845703125,
    -0.00015234947204589844,
    -1.3662109375,
    -2.990234375,
    -0.78271484375,
    -1.4150390625,
    -2.05859375,
    -1.1865234375,
    -1.3525390625,
    -0.8310546875,
    -1.806640625,
    -1.18359375,
    -1.388671875,
    -0.048675537109375,
    -0.176513671875,
    -0.19482421875,
    1.1728515625,
    1.22265625,
    -0.304931640625,
    -0.265869140625,
    -0.537109375,
    -0.06768798828125,
    -1.03515625,
    0.77294921875,
    0.81640625,
    1.2431640625,
    -0.494873046875,
    -0.29052734375,
    -0.890625,
    0.51513671875,
    0.50439453125,
    -0.77685546875,
    0.01084136962890625,
    -0.03497314453125,
    1.0302734375,
    0.468994140625,
    -0.7626953125,
    -0.5546875,
    1.9072265625,
    -0.347900390625,
    -0.6904296875,
    -0.203125,
    0.4462890625,
    -0.08306884765625,
    -6.24609375
   ],
   "raw256_argmax": 2,
   "option_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "p_full": [
    0.005876559764146805,
    0.00313235423527658,
    0.9908550977706909,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00013602394028566778
   ],
   "p": [
    0.005877359304577112,
    0.0031327803153544664,
    0.9909898638725281
   ],
   "p_oracle": [
    0.0057459608651697636,
    0.003123035654425621,
    0.9911310076713562
   ],
   "abstain": 0.00013602394028566778,
   "abstain_oracle": 0.0001346878707408905,
   "abs_delta_abstain": 1.3360695447772741e-06,
   "oracle_margin": 0.9853850603103638,
   "max_abs_delta_p": 0.000141143798828125,
   "mean_abs_delta_p": 9.409563305477302e-05,
   "max_abs_delta_raw_logits": 0.03434467315673828,
   "mean_abs_delta_raw_logits": 0.0056269589986186475,
   "logits_min": -7.8515625,
   "logits_max": 3.16796875,
   "raw_logits_min": -7.8515625,
   "raw_logits_max": 3.16796875,
   "raw_logits_absmax": 7.8515625,
   "oracle_logits_min": -7.847484588623047,
   "oracle_logits_max": 3.175899028778076,
   "oracle_raw_logits_absmax": 7.847484588623047,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.43962979200296104,
   "seconds": 0.43962979200296104,
   "full_vocab_argmax_id": 2,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r02-cold",
   "request_id": "r02",
   "question_id": "cold",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 70,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -1.7734375,
    0.263671875,
    -1.0634765625,
    -1.36328125,
    -1.1494140625,
    -2.90234375,
    -1.9794921875,
    -2.763671875,
    -3.564453125,
    -1.7060546875,
    -2.130859375,
    -3.451171875,
    -4.1328125,
    -2.55859375,
    -2.892578125,
    -3.86328125,
    -3.642578125,
    -4.97265625,
    -2.841796875,
    -2.61328125,
    -3.375,
    -1.7431640625,
    -4.8984375,
    -5.92578125,
    -3.48046875,
    -4.42578125,
    -2.44140625,
    -4.82421875,
    -3.484375,
    -5.1171875,
    -4.00390625,
    -3.8984375,
    -5.44921875,
    -3.9375,
    -2.458984375,
    -4.421875,
    -4.87890625,
    -5.53125,
    -4.3671875,
    -2.6484375,
    -6.53515625,
    -3.927734375,
    -4.484375,
    -5.6328125,
    -3.154296875,
    -4.2734375,
    -4.0078125,
    -4.3046875,
    -4.44140625,
    -4.14453125,
    -5.0,
    -4.0625,
    -2.646484375,
    -4.75390625,
    -2.466796875,
    -3.337890625,
    -3.51171875,
    -4.39453125,
    -4.0078125,
    -5.23046875,
    -6.92578125,
    -6.01953125,
    -7.52734375,
    -5.71875,
    -3.173828125,
    -7.171875,
    -6.96875,
    -2.09765625,
    -7.25,
    -4.0625,
    -7.375,
    -4.83203125,
    -2.16015625,
    -5.3125,
    -3.580078125,
    -5.5859375,
    -4.171875,
    -6.10546875,
    -3.541015625,
    -5.3671875,
    -3.876953125,
    -3.419921875,
    -2.3828125,
    -3.8671875,
    -3.126953125,
    -3.693359375,
    -2.4375,
    -3.1875,
    -2.869140625,
    -3.966796875,
    -3.125,
    -3.60546875,
    -3.166015625,
    -3.888671875,
    -4.25,
    -4.48046875,
    -4.4609375,
    -3.240234375,
    -2.900390625,
    -3.947265625,
    -5.01171875,
    -4.69140625,
    -2.759765625,
    -3.431640625,
    -4.09375,
    -5.25390625,
    -3.0234375,
    -4.55859375,
    -3.748046875,
    -4.9609375,
    -4.05859375,
    -4.203125,
    -4.328125,
    -5.59765625,
    -4.16015625,
    -3.23828125,
    -4.50390625,
    -5.0703125,
    -3.837890625,
    -2.974609375,
    -3.001953125,
    -4.05078125,
    -2.888671875,
    -3.3046875,
    -2.17578125,
    -4.92578125,
    -4.80859375,
    -2.55078125,
    -2.697265625,
    -4.6171875,
    -5.109375,
    -2.564453125,
    -5.10546875,
    -5.6953125,
    -5.0,
    -4.14453125,
    -3.359375,
    -4.27734375,
    -5.3125,
    -2.05859375,
    -3.140625,
    -3.771484375,
    -3.154296875,
    -3.013671875,
    -4.15234375,
    -4.6328125,
    -5.3359375,
    -4.7734375,
    -4.78125,
    -3.982421875,
    -4.12890625,
    -2.67578125,
    -0.8212890625,
    -2.60546875,
    -2.001953125,
    -1.072265625,
    -1.421875,
    -1.5732421875,
    -2.8359375,
    -0.5439453125,
    -1.7060546875,
    -1.8798828125,
    -0.89111328125,
    -1.046875,
    -2.8046875,
    -2.11328125,
    -2.189453125,
    -1.62890625,
    -3.326171875,
    -1.8984375,
    -1.7529296875,
    -1.95703125,
    -1.64453125,
    -2.681640625,
    -0.74072265625,
    -1.4736328125,
    -2.943359375,
    -3.017578125,
    -2.515625,
    -2.677734375,
    -2.830078125,
    -1.9716796875,
    -2.310546875,
    -1.66015625,
    -2.0,
    -1.4208984375,
    -1.302734375,
    -1.279296875,
    -1.6044921875,
    -1.634765625,
    -0.4609375,
    -1.44921875,
    -2.197265625,
    -1.0517578125,
    -1.421875,
    -1.64453125,
    -1.7353515625,
    -1.125,
    -1.544921875,
    -0.9228515625,
    -0.96533203125,
    -1.41796875,
    -1.2451171875,
    -1.40625,
    -1.8896484375,
    -0.4970703125,
    -2.26953125,
    -2.484375,
    -0.01441192626953125,
    -1.5458984375,
    -0.415771484375,
    -0.7744140625,
    -0.16748046875,
    -0.8271484375,
    -2.322265625,
    -0.708984375,
    -1.279296875,
    -2.107421875,
    -1.2939453125,
    -1.3388671875,
    -1.3232421875,
    -1.6328125,
    -1.369140625,
    -1.0205078125,
    -0.133056640625,
    0.3623046875,
    -0.1890869140625,
    0.962890625,
    1.080078125,
    -0.41845703125,
    -0.493896484375,
    -0.315673828125,
    -0.2274169921875,
    -0.80224609375,
    0.493408203125,
    1.1484375,
    1.4677734375,
    -0.56591796875,
    -0.2481689453125,
    -0.58984375,
    0.53955078125,
    0.71484375,
    -0.496826171875,
    0.09527587890625,
    0.00719451904296875,
    0.28271484375,
    0.40673828125,
    -0.63720703125,
    -0.82080078125,
    1.623046875,
    -0.1671142578125,
    -0.80810546875,
    -0.168701171875,
    0.51806640625,
    -0.72119140625,
    -8.4296875
   ],
   "raw256_argmax": 249,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.11674784868955612,
    0.8830952048301697,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00015700262156315148
   ],
   "p": [
    0.11676617711782455,
    0.8832338452339172
   ],
   "p_oracle": [
    0.11606618016958237,
    0.8839338421821594
   ],
   "abstain": 0.00015700262156315148,
   "abstain_oracle": 0.0001577989023644477,
   "abs_delta_abstain": 7.962808012962341e-07,
   "oracle_margin": 0.7678676843643188,
   "max_abs_delta_p": 0.0006999969482421875,
   "mean_abs_delta_p": 0.0006999969482421875,
   "max_abs_delta_raw_logits": 0.0255889892578125,
   "mean_abs_delta_raw_logits": 0.004949908150592819,
   "logits_min": -8.4296875,
   "logits_max": 1.623046875,
   "raw_logits_min": -8.4296875,
   "raw_logits_max": 1.623046875,
   "raw_logits_absmax": 8.4296875,
   "oracle_logits_min": -8.42408561706543,
   "oracle_logits_max": 1.6328606605529785,
   "oracle_raw_logits_absmax": 8.42408561706543,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.39258958399295807,
   "seconds": 0.39258958399295807,
   "full_vocab_argmax_id": 249,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r03-color",
   "request_id": "r03",
   "question_id": "color",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 85,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.25,
    3.552734375,
    -2.693359375,
    -2.67578125,
    -2.1015625,
    -2.888671875,
    -2.900390625,
    -3.86328125,
    -3.939453125,
    -2.744140625,
    -3.36328125,
    -4.10546875,
    -5.42578125,
    -3.51171875,
    -4.5,
    -4.84375,
    -4.09765625,
    -4.26953125,
    -3.072265625,
    -3.115234375,
    -3.37890625,
    -2.052734375,
    -5.1875,
    -7.03125,
    -3.92578125,
    -5.32421875,
    -2.513671875,
    -5.53515625,
    -4.5078125,
    -7.0234375,
    -4.62109375,
    -4.96875,
    -6.30859375,
    -5.3125,
    -3.3203125,
    -5.5546875,
    -5.390625,
    -6.02734375,
    -4.2109375,
    -3.19921875,
    -7.1640625,
    -4.421875,
    -4.84375,
    -6.19140625,
    -4.6953125,
    -5.2109375,
    -4.609375,
    -4.53515625,
    -5.24609375,
    -4.6484375,
    -5.83984375,
    -4.8515625,
    -2.70703125,
    -4.9921875,
    -2.8203125,
    -3.275390625,
    -3.697265625,
    -4.80078125,
    -4.98046875,
    -5.89453125,
    -7.7109375,
    -6.46484375,
    -7.7265625,
    -6.70703125,
    -3.6328125,
    -7.55859375,
    -7.70703125,
    -2.44140625,
    -7.875,
    -4.0859375,
    -8.078125,
    -5.48828125,
    -2.771484375,
    -6.2265625,
    -3.87109375,
    -6.34765625,
    -4.32421875,
    -6.96875,
    -4.2734375,
    -5.7890625,
    -4.38671875,
    -4.28515625,
    -2.7734375,
    -4.21484375,
    -3.568359375,
    -4.078125,
    -2.623046875,
    -3.84375,
    -3.111328125,
    -4.99609375,
    -3.541015625,
    -4.203125,
    -4.375,
    -4.1328125,
    -4.63671875,
    -4.87890625,
    -5.33203125,
    -4.08984375,
    -4.34765625,
    -4.2578125,
    -6.25,
    -4.5625,
    -3.697265625,
    -4.2578125,
    -4.640625,
    -6.2421875,
    -4.4765625,
    -5.859375,
    -4.328125,
    -5.66015625,
    -4.5,
    -4.87109375,
    -5.46875,
    -6.7265625,
    -4.89453125,
    -4.35546875,
    -4.65234375,
    -6.11328125,
    -4.41015625,
    -3.53515625,
    -3.50390625,
    -4.0859375,
    -3.330078125,
    -3.958984375,
    -2.595703125,
    -6.171875,
    -5.91015625,
    -3.40234375,
    -3.30078125,
    -5.6875,
    -5.76171875,
    -3.880859375,
    -5.75390625,
    -6.00390625,
    -5.01171875,
    -4.75390625,
    -4.203125,
    -5.18359375,
    -5.921875,
    -3.27734375,
    -3.69921875,
    -4.26171875,
    -3.94921875,
    -3.24609375,
    -5.0390625,
    -5.5859375,
    -6.21875,
    -5.26171875,
    -4.875,
    -4.703125,
    -5.0,
    -3.375,
    -1.2314453125,
    -3.55859375,
    -2.681640625,
    -1.828125,
    -2.0703125,
    -2.138671875,
    -3.5625,
    -0.41015625,
    -2.27734375,
    -2.18359375,
    -1.9482421875,
    -1.2861328125,
    -2.63671875,
    -2.4453125,
    -3.0234375,
    -1.990234375,
    -3.55078125,
    -2.017578125,
    -2.458984375,
    -2.625,
    -2.681640625,
    -3.1875,
    -1.025390625,
    -2.06640625,
    -3.47265625,
    -3.47265625,
    -3.208984375,
    -3.548828125,
    -3.560546875,
    -3.455078125,
    -2.68359375,
    -2.14453125,
    -2.66015625,
    -1.6328125,
    -1.2177734375,
    -1.8115234375,
    -1.9501953125,
    -2.283203125,
    -1.0517578125,
    -1.8330078125,
    -3.021484375,
    -1.9228515625,
    -1.5185546875,
    -2.240234375,
    -2.04296875,
    -2.01171875,
    -1.765625,
    -1.474609375,
    -1.2783203125,
    -1.95703125,
    -1.990234375,
    -1.9541015625,
    -2.544921875,
    -1.25,
    -2.865234375,
    -2.5859375,
    -0.455322265625,
    -2.4453125,
    -0.9208984375,
    -1.4267578125,
    -0.9619140625,
    -1.7158203125,
    -2.5234375,
    -1.3486328125,
    -1.9931640625,
    -2.4609375,
    -1.82421875,
    -2.048828125,
    -1.3203125,
    -1.669921875,
    -1.296875,
    -1.3974609375,
    -0.1427001953125,
    0.07269287109375,
    -0.3291015625,
    0.90234375,
    1.0751953125,
    -0.0139923095703125,
    -0.85986328125,
    -0.63134765625,
    0.1971435546875,
    -0.7724609375,
    0.5576171875,
    1.083984375,
    1.44921875,
    -0.65576171875,
    0.06463623046875,
    -0.82421875,
    0.260498046875,
    0.83837890625,
    -0.69287109375,
    -0.1121826171875,
    0.1031494140625,
    0.6337890625,
    0.45068359375,
    -0.2332763671875,
    -0.98779296875,
    1.5068359375,
    -0.07843017578125,
    -0.701171875,
    -0.14111328125,
    0.489013671875,
    -0.43505859375,
    -4.16796875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.004130278714001179,
    0.9924781322479248,
    0.0027169270906597376,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0006746469298377633
   ],
   "p": [
    0.0041330670937895775,
    0.993148148059845,
    0.0027187613304704428
   ],
   "p_oracle": [
    0.004039411433041096,
    0.993238091468811,
    0.0027225520461797714
   ],
   "abstain": 0.0006746469298377633,
   "abstain_oracle": 0.0006879517459310591,
   "abs_delta_abstain": 1.3304816093295813e-05,
   "oracle_margin": 0.9891986846923828,
   "max_abs_delta_p": 9.365566074848175e-05,
   "mean_abs_delta_p": 6.246326180795829e-05,
   "max_abs_delta_raw_logits": 0.02802133560180664,
   "mean_abs_delta_raw_logits": 0.006729929620632902,
   "logits_min": -8.078125,
   "logits_max": 3.552734375,
   "raw_logits_min": -8.078125,
   "raw_logits_max": 3.552734375,
   "raw_logits_absmax": 8.078125,
   "oracle_logits_min": -8.06052017211914,
   "oracle_logits_max": 3.5530474185943604,
   "oracle_raw_logits_absmax": 8.06052017211914,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4767402500147,
   "seconds": 0.4767402500147,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r03-shape",
   "request_id": "r03",
   "question_id": "shape",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 124,
   "nopts": 12,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.044921875,
    -1.40625,
    -2.0390625,
    -3.787109375,
    4.84375,
    -1.78515625,
    -1.884765625,
    -3.53515625,
    -3.95703125,
    -2.037109375,
    -3.31640625,
    -2.45703125,
    -3.953125,
    -3.16015625,
    -4.2890625,
    -4.6796875,
    -3.92578125,
    -5.09375,
    -2.4765625,
    -2.791015625,
    -3.548828125,
    -2.10546875,
    -4.734375,
    -6.71484375,
    -3.970703125,
    -4.19140625,
    -2.83203125,
    -5.0390625,
    -3.693359375,
    -6.14453125,
    -4.1484375,
    -4.4453125,
    -6.31640625,
    -4.53125,
    -2.4140625,
    -4.38671875,
    -5.3359375,
    -6.1328125,
    -4.0,
    -2.814453125,
    -7.16796875,
    -3.96875,
    -4.95703125,
    -5.921875,
    -5.1171875,
    -5.21875,
    -4.765625,
    -4.2421875,
    -4.96875,
    -4.19921875,
    -5.23828125,
    -4.2578125,
    -3.080078125,
    -5.4765625,
    -2.810546875,
    -2.9921875,
    -4.1875,
    -5.671875,
    -4.9765625,
    -6.48828125,
    -7.49609375,
    -7.0,
    -7.703125,
    -6.50390625,
    -3.568359375,
    -7.66796875,
    -7.6015625,
    -2.880859375,
    -7.78125,
    -4.3671875,
    -7.6953125,
    -5.1328125,
    -2.6640625,
    -5.40234375,
    -3.625,
    -5.90625,
    -4.5703125,
    -7.12109375,
    -3.71875,
    -5.5859375,
    -4.7109375,
    -4.5859375,
    -2.919921875,
    -4.29296875,
    -3.27734375,
    -4.44140625,
    -2.498046875,
    -3.857421875,
    -3.109375,
    -4.5546875,
    -3.689453125,
    -4.59375,
    -4.296875,
    -4.40625,
    -5.23046875,
    -5.0625,
    -4.984375,
    -3.291015625,
    -3.8125,
    -4.453125,
    -5.41796875,
    -4.4921875,
    -3.115234375,
    -3.525390625,
    -4.7109375,
    -5.27734375,
    -4.078125,
    -5.35546875,
    -4.62890625,
    -6.16015625,
    -3.958984375,
    -4.94921875,
    -4.6171875,
    -6.47265625,
    -4.19140625,
    -3.623046875,
    -4.91796875,
    -5.90625,
    -4.03125,
    -3.291015625,
    -3.51171875,
    -4.08203125,
    -2.966796875,
    -3.755859375,
    -2.244140625,
    -5.96484375,
    -5.4375,
    -2.552734375,
    -3.044921875,
    -4.91796875,
    -6.16796875,
    -3.09765625,
    -5.73828125,
    -6.6484375,
    -5.23046875,
    -4.28515625,
    -3.970703125,
    -4.39453125,
    -5.4609375,
    -2.419921875,
    -3.15625,
    -3.96875,
    -3.466796875,
    -2.505859375,
    -4.39453125,
    -5.53515625,
    -5.76171875,
    -5.69140625,
    -5.37109375,
    -5.0546875,
    -4.86328125,
    -3.771484375,
    -0.396728515625,
    -3.48046875,
    -2.912109375,
    -1.4716796875,
    -1.4111328125,
    -1.845703125,
    -2.94140625,
    0.277099609375,
    -1.8369140625,
    -2.296875,
    -1.8818359375,
    -1.359375,
    -2.73046875,
    -2.37109375,
    -2.484375,
    -2.775390625,
    -3.111328125,
    -2.08203125,
    -2.513671875,
    -2.6171875,
    -2.123046875,
    -3.353515625,
    -1.5751953125,
    -1.552734375,
    -2.890625,
    -3.658203125,
    -3.091796875,
    -2.255859375,
    -2.904296875,
    -2.669921875,
    -2.9140625,
    -1.6806640625,
    -2.943359375,
    -1.845703125,
    -1.978515625,
    -1.939453125,
    -1.9990234375,
    -2.494140625,
    -0.703125,
    -1.697265625,
    -1.958984375,
    -1.9716796875,
    -2.046875,
    -1.7890625,
    -1.8330078125,
    -1.84765625,
    -1.9736328125,
    -1.107421875,
    -1.0849609375,
    -1.5888671875,
    -1.9619140625,
    -2.1171875,
    -2.359375,
    -0.9296875,
    -2.42578125,
    -2.1484375,
    -0.1678466796875,
    -1.962890625,
    -0.58154296875,
    -0.97216796875,
    -0.83642578125,
    -1.2763671875,
    -2.978515625,
    -0.93017578125,
    -2.208984375,
    -1.9033203125,
    -1.1142578125,
    -1.15625,
    -0.7451171875,
    -1.978515625,
    -0.9755859375,
    -1.2822265625,
    -0.34521484375,
    -0.0811767578125,
    0.094482421875,
    1.197265625,
    1.771484375,
    -0.0208282470703125,
    -0.66650390625,
    -0.69287109375,
    0.0341796875,
    -0.7734375,
    0.81982421875,
    0.7060546875,
    1.5654296875,
    -0.7373046875,
    -0.38916015625,
    -0.79736328125,
    0.59716796875,
    0.6044921875,
    -1.2431640625,
    -0.383544921875,
    -0.244140625,
    0.93994140625,
    0.59228515625,
    -0.8984375,
    -1.1220703125,
    1.603515625,
    -0.314453125,
    -0.301513671875,
    -0.361083984375,
    0.5927734375,
    -0.2025146484375,
    -5.93359375
   ],
   "raw256_argmax": 4,
   "option_argmax": 4,
   "oracle_argmax": 4,
   "argmax_agrees": true,
   "p_full": [
    0.0037781158462166786,
    0.0026855291798710823,
    0.0014770656125620008,
    0.0002832776226568967,
    0.9846349358558655,
    0.001877467380836606,
    0.0017088529421016574,
    0.0003594046283978969,
    0.00024126650532707572,
    0.0014797939220443368,
    0.00044190851622261107,
    0.000995212933048606,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    3.728672527358867e-05
   ],
   "p": [
    0.0037782564759254456,
    0.00268562906421721,
    0.001477120560593903,
    0.00028328815824352205,
    0.9846715331077576,
    0.0018775372300297022,
    0.001708916504867375,
    0.0003594179870560765,
    0.00024127548385877162,
    0.001479848986491561,
    0.0004419249598868191,
    0.0009952499531209469
   ],
   "p_oracle": [
    0.0036773886531591415,
    0.0026007944252341986,
    0.0014550749910995364,
    0.0002769312704913318,
    0.9849311113357544,
    0.0018732596654444933,
    0.0016641964903101325,
    0.00036053467192687094,
    0.00023951621551532298,
    0.001478387275710702,
    0.0004451809509191662,
    0.0009976954897865653
   ],
   "abstain": 3.728672527358867e-05,
   "abstain_oracle": 3.644650860223919e-05,
   "abs_delta_abstain": 8.402166713494807e-07,
   "oracle_margin": 0.9812537431716919,
   "max_abs_delta_p": 0.00025957822799682617,
   "mean_abs_delta_p": 4.439332648568476e-05,
   "max_abs_delta_raw_logits": 0.023746013641357422,
   "mean_abs_delta_raw_logits": 0.005930147570325062,
   "logits_min": -7.78125,
   "logits_max": 4.84375,
   "raw_logits_min": -7.78125,
   "raw_logits_max": 4.84375,
   "raw_logits_absmax": 7.78125,
   "oracle_logits_min": -7.787149429321289,
   "oracle_logits_max": 4.8594279289245605,
   "oracle_raw_logits_absmax": 7.787149429321289,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.6980460829799995,
   "seconds": 0.6980460829799995,
   "full_vocab_argmax_id": 4,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r03-wet",
   "request_id": "r03",
   "question_id": "wet",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 79,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    2.65234375,
    -1.4345703125,
    -1.626953125,
    -2.4765625,
    -2.369140625,
    -2.818359375,
    -2.763671875,
    -3.69140625,
    -3.001953125,
    -1.978515625,
    -2.310546875,
    -3.3203125,
    -3.71484375,
    -2.255859375,
    -3.365234375,
    -4.1171875,
    -4.01171875,
    -4.80078125,
    -2.751953125,
    -3.23828125,
    -3.369140625,
    -1.6630859375,
    -4.98828125,
    -5.94921875,
    -3.220703125,
    -4.0078125,
    -2.498046875,
    -4.52734375,
    -3.748046875,
    -6.08203125,
    -3.78515625,
    -4.7421875,
    -5.6484375,
    -4.125,
    -2.78125,
    -4.71875,
    -4.84375,
    -5.8984375,
    -4.3984375,
    -3.03125,
    -6.42578125,
    -4.25,
    -4.8359375,
    -5.76953125,
    -3.904296875,
    -4.26171875,
    -3.86328125,
    -4.51953125,
    -4.921875,
    -4.0078125,
    -5.15234375,
    -4.18359375,
    -3.224609375,
    -5.109375,
    -3.048828125,
    -3.3515625,
    -3.88671875,
    -4.48046875,
    -4.40625,
    -6.0234375,
    -7.015625,
    -6.70703125,
    -7.76953125,
    -5.54296875,
    -3.6484375,
    -6.98046875,
    -7.17578125,
    -2.34375,
    -7.171875,
    -4.06640625,
    -7.54296875,
    -4.8203125,
    -2.8671875,
    -5.25,
    -3.462890625,
    -5.875,
    -4.01953125,
    -6.59375,
    -3.671875,
    -5.63671875,
    -4.33203125,
    -3.673828125,
    -2.64453125,
    -4.27734375,
    -3.150390625,
    -4.2265625,
    -2.224609375,
    -3.83984375,
    -3.0546875,
    -4.39453125,
    -3.390625,
    -3.97265625,
    -3.888671875,
    -3.3828125,
    -4.37890625,
    -4.15625,
    -4.69140625,
    -3.287109375,
    -2.828125,
    -4.03125,
    -5.1171875,
    -4.7265625,
    -3.15234375,
    -3.6328125,
    -4.20703125,
    -5.4921875,
    -3.587890625,
    -4.921875,
    -4.03515625,
    -5.609375,
    -4.33203125,
    -4.55078125,
    -4.84375,
    -5.671875,
    -4.6484375,
    -3.71875,
    -4.578125,
    -5.4296875,
    -3.685546875,
    -3.474609375,
    -3.09765625,
    -4.21484375,
    -3.015625,
    -3.388671875,
    -2.240234375,
    -5.08203125,
    -5.375,
    -2.533203125,
    -2.70703125,
    -5.20703125,
    -5.5546875,
    -2.458984375,
    -5.25390625,
    -6.0,
    -4.87109375,
    -3.951171875,
    -3.470703125,
    -4.70703125,
    -5.53515625,
    -2.830078125,
    -3.287109375,
    -4.08984375,
    -3.205078125,
    -2.529296875,
    -4.7109375,
    -4.7734375,
    -5.421875,
    -5.0703125,
    -5.08203125,
    -4.25390625,
    -4.25390625,
    -2.576171875,
    -1.0234375,
    -3.23828125,
    -2.69140625,
    -1.2734375,
    -1.7890625,
    -1.828125,
    -2.828125,
    -0.61083984375,
    -1.71875,
    -2.048828125,
    -1.58203125,
    -0.98681640625,
    -2.8359375,
    -1.728515625,
    -2.537109375,
    -2.14453125,
    -3.697265625,
    -1.822265625,
    -1.9755859375,
    -2.435546875,
    -2.025390625,
    -2.900390625,
    -0.64794921875,
    -1.990234375,
    -3.451171875,
    -3.244140625,
    -2.71484375,
    -2.212890625,
    -3.09765625,
    -2.240234375,
    -2.7265625,
    -1.9794921875,
    -2.24609375,
    -1.783203125,
    -1.4384765625,
    -1.2333984375,
    -2.15234375,
    -2.48046875,
    -0.3798828125,
    -1.591796875,
    -2.408203125,
    -1.4580078125,
    -1.4072265625,
    -1.716796875,
    -2.095703125,
    -1.60546875,
    -1.8759765625,
    -0.9931640625,
    -1.0849609375,
    -1.7265625,
    -1.61328125,
    -1.666015625,
    -2.375,
    -0.94970703125,
    -2.1953125,
    -2.326171875,
    0.0626220703125,
    -1.8447265625,
    -1.2578125,
    -1.0400390625,
    0.0199737548828125,
    -0.59716796875,
    -2.533203125,
    -0.8916015625,
    -1.515625,
    -1.8994140625,
    -1.123046875,
    -1.7978515625,
    -1.0107421875,
    -1.7431640625,
    -0.84423828125,
    -1.0888671875,
    -0.21240234375,
    -0.0662841796875,
    -0.61181640625,
    0.98876953125,
    0.876953125,
    -0.12841796875,
    -0.27587890625,
    -0.60595703125,
    0.4228515625,
    -0.83447265625,
    0.47509765625,
    0.6474609375,
    1.4091796875,
    -0.308837890625,
    0.0950927734375,
    -0.84521484375,
    0.33642578125,
    0.59521484375,
    -0.69482421875,
    -0.266357421875,
    -0.19140625,
    0.63623046875,
    0.56884765625,
    -0.63818359375,
    -0.96923828125,
    2.1953125,
    -0.306396484375,
    -0.313720703125,
    -0.237548828125,
    0.439697265625,
    -0.498046875,
    -5.56640625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9827590584754944,
    0.016961073502898216,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00027995050186291337
   ],
   "p": [
    0.9830341339111328,
    0.016965821385383606
   ],
   "p_oracle": [
    0.9831133484840393,
    0.016886629164218903
   ],
   "abstain": 0.00027995050186291337,
   "abstain_oracle": 0.0002791435399558395,
   "abs_delta_abstain": 8.069619070738554e-07,
   "oracle_margin": 0.9662266969680786,
   "max_abs_delta_p": 7.921457290649414e-05,
   "mean_abs_delta_p": 7.920339703559875e-05,
   "max_abs_delta_raw_logits": 0.030258655548095703,
   "mean_abs_delta_raw_logits": 0.005116131855174899,
   "logits_min": -7.76953125,
   "logits_max": 2.65234375,
   "raw_logits_min": -7.76953125,
   "raw_logits_max": 2.65234375,
   "raw_logits_absmax": 7.76953125,
   "oracle_logits_min": -7.771839141845703,
   "oracle_logits_max": 2.6563339233398438,
   "oracle_raw_logits_absmax": 7.771839141845703,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4437297909753397,
   "seconds": 0.4437297909753397,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r04-status",
   "request_id": "r04",
   "question_id": "status",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 106,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    3.265625,
    -1.5537109375,
    -2.8203125,
    -2.64453125,
    -2.369140625,
    -2.59375,
    -2.5234375,
    -3.8671875,
    -3.64453125,
    -2.466796875,
    -2.623046875,
    -3.185546875,
    -4.38671875,
    -2.412109375,
    -3.4453125,
    -3.861328125,
    -3.96875,
    -4.8203125,
    -3.025390625,
    -3.37890625,
    -3.337890625,
    -1.8583984375,
    -5.171875,
    -5.93359375,
    -3.689453125,
    -4.27734375,
    -2.671875,
    -4.703125,
    -3.84375,
    -6.1875,
    -4.06640625,
    -4.4765625,
    -5.75,
    -4.06640625,
    -3.091796875,
    -4.5390625,
    -4.98828125,
    -5.96875,
    -4.00390625,
    -3.0859375,
    -6.15625,
    -4.30078125,
    -5.0859375,
    -5.69140625,
    -3.642578125,
    -4.515625,
    -3.7734375,
    -4.66796875,
    -5.0,
    -4.109375,
    -5.18359375,
    -4.21875,
    -3.0546875,
    -5.28515625,
    -3.1015625,
    -3.361328125,
    -3.951171875,
    -4.6640625,
    -4.49609375,
    -6.09375,
    -7.1796875,
    -6.80078125,
    -7.79296875,
    -5.671875,
    -3.876953125,
    -7.01171875,
    -7.3046875,
    -2.4140625,
    -7.2109375,
    -3.927734375,
    -7.7734375,
    -5.02734375,
    -2.865234375,
    -5.30078125,
    -3.69140625,
    -5.83203125,
    -4.1328125,
    -6.73828125,
    -3.63671875,
    -5.69140625,
    -4.28515625,
    -3.904296875,
    -2.71484375,
    -4.19140625,
    -3.35546875,
    -4.3828125,
    -2.39453125,
    -3.810546875,
    -3.021484375,
    -4.41796875,
    -3.634765625,
    -4.109375,
    -4.00390625,
    -3.46875,
    -4.16796875,
    -4.14453125,
    -4.765625,
    -3.369140625,
    -2.91015625,
    -4.05859375,
    -5.32421875,
    -4.5859375,
    -3.03515625,
    -3.958984375,
    -4.4296875,
    -5.5703125,
    -3.533203125,
    -4.95703125,
    -4.0703125,
    -5.7734375,
    -4.1796875,
    -4.58203125,
    -4.80859375,
    -5.91015625,
    -4.578125,
    -3.7109375,
    -4.5546875,
    -5.4765625,
    -3.783203125,
    -3.748046875,
    -2.9765625,
    -4.1953125,
    -2.91015625,
    -3.6171875,
    -2.166015625,
    -5.06640625,
    -5.5,
    -2.412109375,
    -2.7109375,
    -5.3984375,
    -5.65625,
    -2.830078125,
    -5.421875,
    -6.19140625,
    -4.96875,
    -4.09375,
    -3.591796875,
    -4.54296875,
    -5.86328125,
    -2.66796875,
    -3.5078125,
    -4.234375,
    -3.158203125,
    -2.466796875,
    -4.81640625,
    -5.02734375,
    -5.64453125,
    -5.40234375,
    -5.0546875,
    -4.17578125,
    -4.41796875,
    -2.689453125,
    -0.8544921875,
    -3.234375,
    -2.55078125,
    -1.6103515625,
    -1.5546875,
    -1.6884765625,
    -2.876953125,
    -0.5517578125,
    -1.7548828125,
    -2.08203125,
    -1.5673828125,
    -0.982421875,
    -2.740234375,
    -1.8779296875,
    -2.59765625,
    -2.287109375,
    -3.57421875,
    -1.953125,
    -2.1328125,
    -2.462890625,
    -2.08984375,
    -2.955078125,
    -0.62109375,
    -1.7822265625,
    -3.373046875,
    -3.275390625,
    -2.921875,
    -2.24609375,
    -3.375,
    -2.31640625,
    -2.763671875,
    -2.17578125,
    -2.357421875,
    -1.91796875,
    -1.298828125,
    -1.3974609375,
    -2.421875,
    -2.556640625,
    -0.5615234375,
    -1.685546875,
    -2.453125,
    -1.466796875,
    -1.576171875,
    -1.5849609375,
    -2.173828125,
    -1.494140625,
    -2.08203125,
    -0.7958984375,
    -1.0361328125,
    -1.8681640625,
    -1.3115234375,
    -1.7578125,
    -2.248046875,
    -1.1318359375,
    -2.142578125,
    -2.462890625,
    0.049224853515625,
    -1.94140625,
    -1.396484375,
    -1.0029296875,
    -0.023223876953125,
    -0.6943359375,
    -2.798828125,
    -0.92138671875,
    -1.7333984375,
    -1.8935546875,
    -1.05078125,
    -2.0234375,
    -1.3134765625,
    -1.6826171875,
    -0.8203125,
    -0.818359375,
    -0.162353515625,
    -0.0784912109375,
    -0.57763671875,
    0.98974609375,
    0.8896484375,
    -0.189697265625,
    -0.472412109375,
    -0.63525390625,
    0.350830078125,
    -0.83447265625,
    0.1640625,
    0.787109375,
    1.267578125,
    -0.1680908203125,
    0.060882568359375,
    -0.658203125,
    0.22607421875,
    0.77001953125,
    -0.64453125,
    -0.195556640625,
    -0.2568359375,
    0.65966796875,
    0.5361328125,
    -0.5419921875,
    -0.69189453125,
    2.203125,
    -0.337646484375,
    -0.2705078125,
    -0.283447265625,
    0.0760498046875,
    -0.39990234375,
    -4.19140625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9856159687042236,
    0.010385655798017979,
    0.003138873027637601,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.000859492807649076
   ],
   "p": [
    0.9864638447761536,
    0.010394589975476265,
    0.003141573164612055
   ],
   "p_oracle": [
    0.986133873462677,
    0.010701395571231842,
    0.0031647749710828066
   ],
   "abstain": 0.000859492807649076,
   "abstain_oracle": 0.0008616404957138002,
   "abs_delta_abstain": 2.147688064724207e-06,
   "oracle_margin": 0.9754324555397034,
   "max_abs_delta_p": 0.0003299713134765625,
   "mean_abs_delta_p": 0.00021999290523429713,
   "max_abs_delta_raw_logits": 0.02630901336669922,
   "mean_abs_delta_raw_logits": 0.005995114421239123,
   "logits_min": -7.79296875,
   "logits_max": 3.265625,
   "raw_logits_min": -7.79296875,
   "raw_logits_max": 3.265625,
   "raw_logits_absmax": 7.79296875,
   "oracle_logits_min": -7.7877583503723145,
   "oracle_logits_max": 3.259012222290039,
   "oracle_raw_logits_absmax": 7.7877583503723145,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5969952919986099,
   "seconds": 0.5969952919986099,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r04-drawer",
   "request_id": "r04",
   "question_id": "drawer",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 103,
   "nopts": 11,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.56640625,
    -1.7060546875,
    -2.03515625,
    -2.53515625,
    -2.369140625,
    -2.697265625,
    5.17578125,
    -5.29296875,
    -2.62109375,
    -2.748046875,
    -3.3671875,
    -3.814453125,
    -4.8515625,
    -2.544921875,
    -3.9921875,
    -5.01171875,
    -4.890625,
    -4.765625,
    -2.498046875,
    -2.685546875,
    -3.78515625,
    -1.908203125,
    -5.21875,
    -6.30078125,
    -4.63671875,
    -5.05078125,
    -2.544921875,
    -5.16796875,
    -4.41796875,
    -6.2890625,
    -4.86328125,
    -4.92578125,
    -5.78515625,
    -4.55859375,
    -2.96484375,
    -4.55859375,
    -5.53515625,
    -6.7109375,
    -3.419921875,
    -3.962890625,
    -6.3125,
    -4.328125,
    -4.859375,
    -5.05078125,
    -4.38671875,
    -4.55078125,
    -5.19140625,
    -5.12890625,
    -5.6171875,
    -3.4453125,
    -5.72265625,
    -4.37109375,
    -1.671875,
    -5.26953125,
    -3.162109375,
    -3.6796875,
    -4.44921875,
    -4.74609375,
    -4.77734375,
    -6.16015625,
    -7.4453125,
    -6.40625,
    -7.5859375,
    -5.80859375,
    -3.814453125,
    -7.3671875,
    -7.5234375,
    -3.0390625,
    -7.8515625,
    -3.93359375,
    -7.55078125,
    -4.9375,
    -2.82421875,
    -6.16796875,
    -3.41796875,
    -5.94921875,
    -4.06640625,
    -6.86328125,
    -3.568359375,
    -5.44921875,
    -4.6015625,
    -4.00390625,
    -2.34765625,
    -4.26171875,
    -3.1484375,
    -4.55859375,
    -2.845703125,
    -4.16796875,
    -3.02734375,
    -4.62109375,
    -3.5390625,
    -4.66015625,
    -4.58984375,
    -4.12890625,
    -5.046875,
    -4.8671875,
    -5.16015625,
    -4.24609375,
    -3.8828125,
    -4.17578125,
    -5.70703125,
    -4.61328125,
    -3.5234375,
    -4.140625,
    -4.30078125,
    -5.5546875,
    -3.708984375,
    -5.39453125,
    -4.28125,
    -5.42578125,
    -3.748046875,
    -4.49609375,
    -5.37890625,
    -6.609375,
    -5.08203125,
    -4.23046875,
    -4.5,
    -5.51171875,
    -4.0,
    -3.404296875,
    -2.935546875,
    -4.30859375,
    -3.294921875,
    -3.828125,
    -2.763671875,
    -5.70703125,
    -5.64453125,
    -2.685546875,
    -2.83984375,
    -5.83984375,
    -5.93359375,
    -3.017578125,
    -5.76953125,
    -6.7109375,
    -5.4140625,
    -4.08203125,
    -3.8828125,
    -4.015625,
    -5.95703125,
    -3.37109375,
    -3.712890625,
    -3.80078125,
    -4.0546875,
    -3.0625,
    -4.7421875,
    -5.3046875,
    -5.87109375,
    -5.14453125,
    -5.29296875,
    -3.869140625,
    -5.44140625,
    -3.376953125,
    -0.9296875,
    -3.74609375,
    -2.2734375,
    -1.6533203125,
    -1.7421875,
    -2.11328125,
    -3.48828125,
    -0.311279296875,
    -1.591796875,
    -2.1953125,
    -1.6904296875,
    -0.95458984375,
    -3.875,
    -2.302734375,
    -2.908203125,
    -2.78515625,
    -3.00390625,
    -1.521484375,
    -2.48046875,
    -2.384765625,
    -1.98046875,
    -2.416015625,
    -0.98046875,
    -2.244140625,
    -3.5625,
    -3.36328125,
    -3.216796875,
    -3.1953125,
    -3.2109375,
    -2.978515625,
    -2.94921875,
    -2.6015625,
    -3.369140625,
    -2.18359375,
    -2.076171875,
    -1.814453125,
    -1.916015625,
    -2.51171875,
    -1.1162109375,
    -1.5791015625,
    -2.81640625,
    -1.8623046875,
    -2.4453125,
    -1.9677734375,
    -2.462890625,
    -1.67578125,
    -1.7744140625,
    -1.5390625,
    -1.5712890625,
    -2.212890625,
    -2.439453125,
    -1.9296875,
    -2.197265625,
    -1.2197265625,
    -2.271484375,
    -2.21875,
    0.197021484375,
    -2.451171875,
    -0.77587890625,
    -0.70263671875,
    -1.03515625,
    -1.4326171875,
    -2.958984375,
    -0.94482421875,
    -2.25390625,
    -2.646484375,
    -1.2685546875,
    -1.6103515625,
    -1.087890625,
    -1.6279296875,
    -1.4423828125,
    -1.041015625,
    -0.41259765625,
    -0.01392364501953125,
    -0.88916015625,
    1.18359375,
    0.7158203125,
    -0.1260986328125,
    -0.42236328125,
    -0.469482421875,
    0.0723876953125,
    -0.74658203125,
    0.404296875,
    1.2734375,
    1.44140625,
    -0.65771484375,
    -0.30078125,
    -0.86376953125,
    1.111328125,
    0.4052734375,
    -0.955078125,
    -0.1231689453125,
    -0.116943359375,
    0.488525390625,
    0.3525390625,
    -0.91748046875,
    -0.84130859375,
    1.1640625,
    -0.028350830078125,
    -0.03631591796875,
    0.11431884765625,
    0.0141754150390625,
    -0.50390625,
    -3.5
   ],
   "raw256_argmax": 6,
   "option_argmax": 6,
   "oracle_argmax": 6,
   "argmax_agrees": true,
   "p_full": [
    0.001699271029792726,
    0.0014892509207129478,
    0.0010913006262853742,
    0.000680463039316237,
    0.0007960078655742109,
    0.0005838409415446222,
    0.9918422698974609,
    5.0272599764866754e-05,
    0.0006274023908190429,
    0.0005564935272559524,
    0.00031005547498352826,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00027349492302164435
   ],
   "p": [
    0.0016997356433421373,
    0.0014896581415086985,
    0.0010915989987552166,
    0.0006806490710005164,
    0.0007962255040183663,
    0.0005840006051585078,
    0.992113471031189,
    5.028634404879995e-05,
    0.0006275739287957549,
    0.0005566456820815802,
    0.0003101402544416487
   ],
   "p_oracle": [
    0.0017093284986913204,
    0.0014980518026277423,
    0.0010982141830027103,
    0.0006639480125159025,
    0.0007912743021734059,
    0.0005999383865855634,
    0.9920872449874878,
    4.973450268153101e-05,
    0.0006263956311158836,
    0.0005665535572916269,
    0.0003093348059337586
   ],
   "abstain": 0.00027349492302164435,
   "abstain_oracle": 0.0002797741908580065,
   "abs_delta_abstain": 6.2792678363621235e-06,
   "oracle_margin": 0.9903779029846191,
   "max_abs_delta_p": 2.6226043701171875e-05,
   "mean_abs_delta_p": 9.1692044489636e-06,
   "max_abs_delta_raw_logits": 0.02786874771118164,
   "mean_abs_delta_raw_logits": 0.006323250592686236,
   "logits_min": -7.8515625,
   "logits_max": 5.17578125,
   "raw_logits_min": -7.8515625,
   "raw_logits_max": 5.17578125,
   "raw_logits_absmax": 7.8515625,
   "oracle_logits_min": -7.854022979736328,
   "oracle_logits_max": 5.175121307373047,
   "oracle_raw_logits_absmax": 7.854022979736328,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5771446670405567,
   "seconds": 0.5771446670405567,
   "full_vocab_argmax_id": 6,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r04-key",
   "request_id": "r04",
   "question_id": "key",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 65,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -3.267578125,
    1.3203125,
    -0.7373046875,
    -1.224609375,
    -1.5146484375,
    -2.71875,
    -2.1640625,
    -3.16015625,
    -3.4609375,
    -1.826171875,
    -2.24609375,
    -3.703125,
    -4.15234375,
    -2.0859375,
    -2.99609375,
    -4.0859375,
    -4.05859375,
    -4.984375,
    -2.8984375,
    -2.607421875,
    -3.50390625,
    -1.7587890625,
    -5.046875,
    -6.39453125,
    -3.720703125,
    -4.484375,
    -2.611328125,
    -4.859375,
    -3.638671875,
    -5.7578125,
    -4.08984375,
    -4.109375,
    -5.5546875,
    -4.23828125,
    -2.458984375,
    -4.921875,
    -5.0,
    -5.78125,
    -4.109375,
    -2.9375,
    -6.703125,
    -4.04296875,
    -4.46875,
    -5.77734375,
    -3.572265625,
    -4.4765625,
    -4.16015625,
    -4.203125,
    -4.83203125,
    -4.171875,
    -5.16015625,
    -4.28125,
    -2.498046875,
    -4.640625,
    -2.34765625,
    -3.19140625,
    -3.423828125,
    -4.421875,
    -4.14453125,
    -5.15234375,
    -6.85546875,
    -5.91015625,
    -7.19140625,
    -5.95703125,
    -3.3046875,
    -7.0625,
    -7.0859375,
    -2.232421875,
    -7.51953125,
    -4.1171875,
    -7.41015625,
    -4.84375,
    -2.3515625,
    -5.70703125,
    -3.626953125,
    -5.640625,
    -4.15234375,
    -6.30859375,
    -3.837890625,
    -5.26953125,
    -4.09375,
    -3.77734375,
    -2.4453125,
    -3.705078125,
    -3.04296875,
    -3.89453125,
    -2.34765625,
    -3.28125,
    -2.71484375,
    -4.3828125,
    -3.462890625,
    -3.71875,
    -3.287109375,
    -3.826171875,
    -4.31640625,
    -4.76171875,
    -4.61328125,
    -3.708984375,
    -3.494140625,
    -3.845703125,
    -5.20703125,
    -4.55859375,
    -3.22265625,
    -3.568359375,
    -4.12890625,
    -5.32421875,
    -3.326171875,
    -4.91015625,
    -3.87890625,
    -5.03125,
    -4.16796875,
    -4.296875,
    -4.5390625,
    -5.828125,
    -4.2890625,
    -3.62890625,
    -4.42578125,
    -5.38671875,
    -3.89453125,
    -3.037109375,
    -3.08203125,
    -3.859375,
    -3.13671875,
    -3.359375,
    -2.42578125,
    -5.296875,
    -4.8828125,
    -2.9296875,
    -2.80078125,
    -4.9375,
    -5.296875,
    -2.984375,
    -5.34375,
    -5.7578125,
    -4.92578125,
    -4.26171875,
    -3.544921875,
    -4.35546875,
    -5.50390625,
    -2.619140625,
    -3.271484375,
    -3.880859375,
    -3.478515625,
    -3.435546875,
    -4.40625,
    -4.52734375,
    -5.44921875,
    -4.8125,
    -4.77734375,
    -3.9765625,
    -4.44921875,
    -2.892578125,
    -0.98193359375,
    -3.009765625,
    -2.130859375,
    -1.369140625,
    -1.6318359375,
    -1.7685546875,
    -2.98828125,
    -0.478271484375,
    -1.990234375,
    -1.98046875,
    -1.34375,
    -1.3271484375,
    -2.80078125,
    -2.177734375,
    -2.5,
    -1.5751953125,
    -3.37109375,
    -1.9736328125,
    -1.6552734375,
    -2.296875,
    -1.998046875,
    -2.703125,
    -0.798828125,
    -1.71484375,
    -2.892578125,
    -3.177734375,
    -2.677734375,
    -3.185546875,
    -2.9921875,
    -2.76171875,
    -2.69140625,
    -1.921875,
    -2.27734375,
    -1.4091796875,
    -1.6611328125,
    -1.603515625,
    -1.6279296875,
    -2.099609375,
    -0.68603515625,
    -1.44140625,
    -2.396484375,
    -1.3759765625,
    -1.5166015625,
    -1.982421875,
    -1.9013671875,
    -1.6484375,
    -1.4365234375,
    -1.2451171875,
    -1.248046875,
    -1.6328125,
    -1.5234375,
    -1.6357421875,
    -2.21484375,
    -0.60595703125,
    -2.4375,
    -2.330078125,
    -0.1317138671875,
    -1.8642578125,
    -0.62255859375,
    -1.0595703125,
    -0.5517578125,
    -0.96484375,
    -2.5,
    -0.994140625,
    -1.5458984375,
    -2.205078125,
    -1.640625,
    -1.5263671875,
    -1.4169921875,
    -1.5791015625,
    -1.4736328125,
    -1.16015625,
    -0.11767578125,
    0.327392578125,
    -0.2232666015625,
    0.94580078125,
    0.990234375,
    -0.128662109375,
    -0.80078125,
    -0.55908203125,
    -0.13671875,
    -0.818359375,
    0.5302734375,
    1.080078125,
    1.5791015625,
    -0.74609375,
    -0.277099609375,
    -0.69677734375,
    0.75048828125,
    0.84912109375,
    -0.62158203125,
    -0.1173095703125,
    -0.16015625,
    0.42529296875,
    0.25439453125,
    -0.78466796875,
    -0.93310546875,
    1.4423828125,
    -0.0765380859375,
    -0.78759765625,
    -0.0206146240234375,
    0.60302734375,
    -0.67578125,
    -6.65234375
   ],
   "raw256_argmax": 236,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.010380245745182037,
    0.9892599582672119,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003598351322580129
   ],
   "p": [
    0.01038398128002882,
    0.9896159768104553
   ],
   "p_oracle": [
    0.010282500647008419,
    0.9897175431251526
   ],
   "abstain": 0.0003598351322580129,
   "abstain_oracle": 0.00035554287023842335,
   "abs_delta_abstain": 4.292262019589543e-06,
   "oracle_margin": 0.9794350266456604,
   "max_abs_delta_p": 0.00010156631469726562,
   "mean_abs_delta_p": 0.00010152347385883331,
   "max_abs_delta_raw_logits": 0.023680925369262695,
   "mean_abs_delta_raw_logits": 0.005000852630473673,
   "logits_min": -7.51953125,
   "logits_max": 1.5791015625,
   "raw_logits_min": -7.51953125,
   "raw_logits_max": 1.5791015625,
   "raw_logits_absmax": 7.51953125,
   "oracle_logits_min": -7.522004127502441,
   "oracle_logits_max": 1.5854170322418213,
   "oracle_raw_logits_absmax": 7.522004127502441,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3677930829580873,
   "seconds": 0.3677930829580873,
   "full_vocab_argmax_id": 236,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r05-task",
   "request_id": "r05",
   "question_id": "task",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 99,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    1.09375,
    -1.5341796875,
    -4.53515625,
    -3.365234375,
    -3.244140625,
    -3.193359375,
    -3.009765625,
    -4.37109375,
    -5.0703125,
    -3.564453125,
    -2.951171875,
    -3.109375,
    -5.2734375,
    -3.4140625,
    -3.943359375,
    -3.90234375,
    -4.28515625,
    -4.734375,
    -3.44140625,
    -3.88671875,
    -3.595703125,
    -2.474609375,
    -5.42578125,
    -5.8671875,
    -4.26953125,
    -5.4765625,
    -2.953125,
    -5.6328125,
    -4.50390625,
    -6.37890625,
    -4.66796875,
    -4.39453125,
    -6.57421875,
    -4.40234375,
    -3.8046875,
    -5.38671875,
    -5.75,
    -5.7421875,
    -4.34375,
    -3.126953125,
    -6.1875,
    -5.2734375,
    -5.5,
    -5.51171875,
    -3.91796875,
    -5.1171875,
    -4.34765625,
    -4.96875,
    -5.41796875,
    -4.6640625,
    -5.53515625,
    -5.046875,
    -3.8203125,
    -5.22265625,
    -3.3359375,
    -3.728515625,
    -4.0234375,
    -4.1640625,
    -4.6796875,
    -5.546875,
    -7.62109375,
    -7.11328125,
    -8.2578125,
    -6.30859375,
    -3.8828125,
    -7.5859375,
    -7.96875,
    -2.86328125,
    -8.1171875,
    -4.19921875,
    -8.0625,
    -5.46484375,
    -2.71875,
    -5.6015625,
    -4.07421875,
    -6.3671875,
    -4.27734375,
    -7.16796875,
    -4.01171875,
    -6.0859375,
    -4.27734375,
    -4.1953125,
    -2.939453125,
    -4.76171875,
    -3.716796875,
    -4.61328125,
    -2.6796875,
    -4.06640625,
    -3.51171875,
    -4.98046875,
    -3.966796875,
    -4.68359375,
    -4.515625,
    -3.86328125,
    -4.6171875,
    -4.046875,
    -5.0625,
    -3.556640625,
    -2.998046875,
    -4.41796875,
    -6.29296875,
    -4.875,
    -3.544921875,
    -4.76171875,
    -5.08984375,
    -6.2421875,
    -4.1328125,
    -5.55078125,
    -4.05859375,
    -5.96875,
    -4.23828125,
    -5.05859375,
    -5.3046875,
    -6.3046875,
    -4.765625,
    -3.58984375,
    -4.703125,
    -6.15625,
    -4.734375,
    -4.01953125,
    -3.14453125,
    -4.12890625,
    -3.0078125,
    -3.931640625,
    -2.216796875,
    -5.44140625,
    -5.9765625,
    -2.732421875,
    -3.103515625,
    -5.96875,
    -5.6953125,
    -3.791015625,
    -6.1015625,
    -6.78125,
    -5.2109375,
    -5.1015625,
    -3.783203125,
    -4.609375,
    -6.4296875,
    -2.939453125,
    -3.98828125,
    -4.43359375,
    -3.302734375,
    -3.103515625,
    -4.328125,
    -5.26953125,
    -6.23828125,
    -5.921875,
    -4.9609375,
    -4.32421875,
    -4.5078125,
    -2.880859375,
    -1.3369140625,
    -3.28515625,
    -2.08203125,
    -1.8193359375,
    -1.4052734375,
    -1.6337890625,
    -2.962890625,
    -1.166015625,
    -2.146484375,
    -1.6953125,
    -1.607421875,
    -1.3408203125,
    -2.384765625,
    -2.2421875,
    -2.984375,
    -2.529296875,
    -3.6953125,
    -2.408203125,
    -2.5234375,
    -2.681640625,
    -2.392578125,
    -3.19921875,
    -1.1767578125,
    -1.9677734375,
    -3.93359375,
    -3.623046875,
    -3.65625,
    -3.13671875,
    -4.6328125,
    -3.15625,
    -3.126953125,
    -3.126953125,
    -3.22265625,
    -2.544921875,
    -1.9423828125,
    -2.23046875,
    -2.8203125,
    -2.705078125,
    -1.6455078125,
    -2.26171875,
    -3.390625,
    -2.044921875,
    -1.8759765625,
    -1.9423828125,
    -2.59375,
    -1.8896484375,
    -2.294921875,
    -1.4296875,
    -1.8056640625,
    -2.453125,
    -1.265625,
    -2.373046875,
    -2.4296875,
    -1.9560546875,
    -2.43359375,
    -2.576171875,
    -0.18212890625,
    -2.650390625,
    -1.7978515625,
    -1.595703125,
    -0.7685546875,
    -1.2861328125,
    -3.171875,
    -1.4716796875,
    -2.46484375,
    -2.369140625,
    -1.8134765625,
    -2.546875,
    -1.791015625,
    -1.240234375,
    -1.2470703125,
    -0.64453125,
    -0.11114501953125,
    -0.2138671875,
    -0.97314453125,
    0.59619140625,
    0.429931640625,
    -0.29248046875,
    -0.2459716796875,
    -0.53759765625,
    -0.02301025390625,
    -0.84716796875,
    -0.2169189453125,
    1.279296875,
    0.99658203125,
    -0.26611328125,
    -0.26904296875,
    -0.70361328125,
    0.37744140625,
    0.6796875,
    -0.496337890625,
    0.05926513671875,
    -0.1573486328125,
    0.491455078125,
    0.7080078125,
    -0.343017578125,
    0.023406982421875,
    1.8876953125,
    -0.6787109375,
    -0.2127685546875,
    -0.50244140625,
    -0.024169921875,
    -0.336669921875,
    -0.61376953125
   ],
   "raw256_argmax": 249,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.7765794992446899,
    0.06486357003450394,
    0.0038085724227130413,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.15474838018417358
   ],
   "p": [
    0.9187554121017456,
    0.07673876732587814,
    0.004505844786763191
   ],
   "p_oracle": [
    0.9156041741371155,
    0.07993196696043015,
    0.004463807214051485
   ],
   "abstain": 0.15474838018417358,
   "abstain_oracle": 0.13591159880161285,
   "abs_delta_abstain": 0.01883678138256073,
   "oracle_margin": 0.8356721997261047,
   "max_abs_delta_p": 0.003193199634552002,
   "mean_abs_delta_p": 0.002128825057297945,
   "max_abs_delta_raw_logits": 0.12987357378005981,
   "mean_abs_delta_raw_logits": 0.021902231208514422,
   "logits_min": -8.2578125,
   "logits_max": 1.8876953125,
   "raw_logits_min": -8.2578125,
   "raw_logits_max": 1.8876953125,
   "raw_logits_absmax": 8.2578125,
   "oracle_logits_min": -8.236438751220703,
   "oracle_logits_max": 1.9041804075241089,
   "oracle_raw_logits_absmax": 8.236438751220703,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5561975000891834,
   "seconds": 0.5561975000891834,
   "full_vocab_argmax_id": 249,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r05-leaves",
   "request_id": "r05",
   "question_id": "leaves",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 71,
   "nopts": 4,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.9228515625,
    -1.7275390625,
    3.669921875,
    -2.458984375,
    -2.646484375,
    -2.4375,
    -2.56640625,
    -3.181640625,
    -2.75,
    -1.9873046875,
    -3.21875,
    -3.03125,
    -4.1875,
    -3.09375,
    -3.51171875,
    -4.41015625,
    -3.9375,
    -4.41796875,
    -2.576171875,
    -3.14453125,
    -3.939453125,
    -1.5625,
    -5.046875,
    -5.890625,
    -3.14453125,
    -4.5546875,
    -3.28125,
    -3.8203125,
    -3.91015625,
    -6.49609375,
    -4.12109375,
    -5.77734375,
    -4.6015625,
    -4.32421875,
    -2.8828125,
    -5.0,
    -5.28125,
    -6.765625,
    -4.85546875,
    -3.21484375,
    -5.8125,
    -4.10546875,
    -4.58203125,
    -6.0625,
    -4.6875,
    -4.91796875,
    -4.5,
    -4.16796875,
    -5.1875,
    -4.9296875,
    -5.20703125,
    -4.92578125,
    -3.099609375,
    -4.83203125,
    -3.099609375,
    -3.65625,
    -3.921875,
    -5.015625,
    -5.140625,
    -6.39453125,
    -7.44140625,
    -6.765625,
    -7.44921875,
    -6.046875,
    -3.455078125,
    -7.0859375,
    -7.37109375,
    -2.8671875,
    -7.8671875,
    -4.609375,
    -7.77734375,
    -5.33984375,
    -2.908203125,
    -5.5625,
    -2.884765625,
    -6.1171875,
    -3.861328125,
    -6.58984375,
    -4.31640625,
    -5.67578125,
    -4.671875,
    -3.73828125,
    -2.625,
    -4.3984375,
    -3.2734375,
    -4.46875,
    -2.552734375,
    -4.17578125,
    -2.978515625,
    -4.45703125,
    -3.662109375,
    -3.8203125,
    -4.33203125,
    -3.326171875,
    -5.00390625,
    -4.6875,
    -5.125,
    -3.373046875,
    -3.21875,
    -4.5,
    -5.65625,
    -4.41796875,
    -3.912109375,
    -3.642578125,
    -4.4296875,
    -5.625,
    -3.486328125,
    -5.3125,
    -4.3359375,
    -5.76171875,
    -3.998046875,
    -4.546875,
    -5.37890625,
    -5.828125,
    -4.38671875,
    -3.75390625,
    -4.453125,
    -5.82421875,
    -3.853515625,
    -3.314453125,
    -3.619140625,
    -4.25390625,
    -3.0078125,
    -3.060546875,
    -2.546875,
    -5.16015625,
    -5.4375,
    -3.05078125,
    -2.533203125,
    -5.25390625,
    -5.2734375,
    -2.55859375,
    -5.43359375,
    -6.09765625,
    -5.2890625,
    -4.72265625,
    -3.55859375,
    -4.4453125,
    -5.7109375,
    -3.400390625,
    -3.0390625,
    -4.0078125,
    -3.498046875,
    -2.857421875,
    -4.14453125,
    -4.5078125,
    -5.4296875,
    -5.015625,
    -4.91796875,
    -4.3828125,
    -4.84765625,
    -2.84375,
    -0.82666015625,
    -3.490234375,
    -2.07421875,
    -1.0009765625,
    -1.6171875,
    -1.7109375,
    -2.873046875,
    0.053558349609375,
    -1.7607421875,
    -1.837890625,
    -1.4814453125,
    -1.5380859375,
    -3.0703125,
    -2.162109375,
    -3.259765625,
    -1.927734375,
    -3.5546875,
    -1.8740234375,
    -2.0703125,
    -2.23828125,
    -1.9697265625,
    -2.646484375,
    -0.62451171875,
    -1.8544921875,
    -2.8359375,
    -3.2890625,
    -2.734375,
    -2.650390625,
    -2.76171875,
    -2.806640625,
    -2.9609375,
    -1.953125,
    -2.724609375,
    -1.751953125,
    -2.369140625,
    -1.2802734375,
    -1.72265625,
    -2.9609375,
    -0.7041015625,
    -1.8017578125,
    -2.64453125,
    -1.6103515625,
    -1.908203125,
    -2.26953125,
    -1.69921875,
    -1.775390625,
    -1.7060546875,
    -1.26953125,
    -1.14453125,
    -2.06640625,
    -1.9775390625,
    -1.88671875,
    -2.71875,
    -0.88037109375,
    -2.54296875,
    -2.310546875,
    -0.07757568359375,
    -2.017578125,
    -0.98388671875,
    -0.998046875,
    -0.2435302734375,
    -1.4189453125,
    -3.025390625,
    -0.853515625,
    -1.5400390625,
    -2.177734375,
    -1.14453125,
    -1.55859375,
    -0.63916015625,
    -1.9833984375,
    -1.078125,
    -1.453125,
    -0.119140625,
    -0.0290374755859375,
    -0.271728515625,
    1.0634765625,
    1.013671875,
    -0.201904296875,
    -0.270263671875,
    -0.56201171875,
    0.016693115234375,
    -0.83056640625,
    0.68310546875,
    0.72412109375,
    1.248046875,
    -0.469970703125,
    0.03668212890625,
    -1.0400390625,
    0.59765625,
    0.37255859375,
    -0.5546875,
    -0.018585205078125,
    -0.032806396484375,
    1.16796875,
    0.5439453125,
    -0.77685546875,
    -0.82470703125,
    1.9599609375,
    -0.40380859375,
    -0.494140625,
    0.057098388671875,
    0.47265625,
    0.01142120361328125,
    -5.609375
   ],
   "raw256_argmax": 2,
   "option_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "p_full": [
    0.005002596881240606,
    0.006016280502080917,
    0.9858127236366272,
    0.0030146106146275997,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00015370614710263908
   ],
   "p": [
    0.005003366153687239,
    0.006017205771058798,
    0.9859643578529358,
    0.0030150741804391146
   ],
   "p_oracle": [
    0.005044578108936548,
    0.00600813515484333,
    0.9859573841094971,
    0.0029899454675614834
   ],
   "abstain": 0.00015370614710263908,
   "abstain_oracle": 0.0001540259108878672,
   "abs_delta_abstain": 3.197637852281332e-07,
   "oracle_margin": 0.9799492359161377,
   "max_abs_delta_p": 4.121195524930954e-05,
   "mean_abs_delta_p": 2.059625694528222e-05,
   "max_abs_delta_raw_logits": 0.027591705322265625,
   "mean_abs_delta_raw_logits": 0.006074239732697606,
   "logits_min": -7.8671875,
   "logits_max": 3.669921875,
   "raw_logits_min": -7.8671875,
   "raw_logits_max": 3.669921875,
   "raw_logits_absmax": 7.8671875,
   "oracle_logits_min": -7.863582611083984,
   "oracle_logits_max": 3.669034242630005,
   "oracle_raw_logits_absmax": 7.863582611083984,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4008094579912722,
   "seconds": 0.4008094579912722,
   "full_vocab_argmax_id": 2,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r05-dry",
   "request_id": "r05",
   "question_id": "dry",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 73,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -0.0214996337890625,
    0.187255859375,
    -1.2783203125,
    -2.208984375,
    -2.109375,
    -2.96484375,
    -2.615234375,
    -3.0078125,
    -4.6015625,
    -2.654296875,
    -1.9697265625,
    -3.34765625,
    -3.837890625,
    -2.427734375,
    -3.12109375,
    -3.947265625,
    -4.01953125,
    -5.33984375,
    -3.328125,
    -3.138671875,
    -3.525390625,
    -2.0703125,
    -5.10546875,
    -6.08984375,
    -3.443359375,
    -4.78125,
    -2.71875,
    -4.7734375,
    -3.650390625,
    -5.42578125,
    -4.03515625,
    -3.953125,
    -5.5,
    -4.03515625,
    -2.65234375,
    -4.69140625,
    -5.21875,
    -5.44921875,
    -4.5703125,
    -2.580078125,
    -6.33984375,
    -4.19921875,
    -4.59375,
    -5.5234375,
    -3.236328125,
    -4.51171875,
    -3.830078125,
    -4.52734375,
    -4.58984375,
    -4.1953125,
    -5.09765625,
    -4.453125,
    -3.0390625,
    -4.7421875,
    -2.7578125,
    -3.44921875,
    -3.6328125,
    -4.40625,
    -4.0625,
    -5.5,
    -7.20703125,
    -6.375,
    -7.8203125,
    -5.83984375,
    -3.287109375,
    -7.1875,
    -7.1953125,
    -2.28515625,
    -7.4921875,
    -4.23828125,
    -7.53515625,
    -4.84765625,
    -2.533203125,
    -5.3671875,
    -3.732421875,
    -5.59375,
    -4.29296875,
    -6.265625,
    -3.603515625,
    -5.44140625,
    -4.13671875,
    -3.498046875,
    -2.388671875,
    -4.04296875,
    -3.29296875,
    -3.94921875,
    -2.470703125,
    -3.392578125,
    -3.068359375,
    -4.1640625,
    -3.150390625,
    -3.724609375,
    -3.38671875,
    -3.767578125,
    -4.359375,
    -4.296875,
    -4.5625,
    -3.35546875,
    -2.64453125,
    -3.87109375,
    -5.171875,
    -4.88671875,
    -3.07421875,
    -3.90625,
    -4.0703125,
    -5.4921875,
    -3.10546875,
    -4.7109375,
    -3.849609375,
    -5.3125,
    -4.14453125,
    -4.5390625,
    -4.39453125,
    -5.62109375,
    -4.2265625,
    -3.171875,
    -4.54296875,
    -5.13671875,
    -3.876953125,
    -3.32421875,
    -3.2109375,
    -4.0703125,
    -2.775390625,
    -3.3828125,
    -2.376953125,
    -4.84765625,
    -5.03125,
    -2.642578125,
    -2.59375,
    -4.71484375,
    -5.17578125,
    -2.607421875,
    -5.0859375,
    -5.875,
    -5.19921875,
    -4.3828125,
    -3.408203125,
    -4.17578125,
    -5.46875,
    -2.291015625,
    -3.33984375,
    -3.943359375,
    -3.009765625,
    -2.970703125,
    -4.328125,
    -4.7265625,
    -5.55078125,
    -4.96875,
    -4.796875,
    -3.8359375,
    -4.125,
    -2.326171875,
    -0.81689453125,
    -2.58203125,
    -1.95703125,
    -0.95703125,
    -1.3876953125,
    -1.560546875,
    -2.736328125,
    -0.75537109375,
    -1.76171875,
    -1.6494140625,
    -0.85107421875,
    -1.1318359375,
    -2.533203125,
    -1.9521484375,
    -2.3671875,
    -1.658203125,
    -3.482421875,
    -1.865234375,
    -1.732421875,
    -2.099609375,
    -1.751953125,
    -2.5859375,
    -0.72265625,
    -1.6171875,
    -3.1796875,
    -3.03125,
    -2.583984375,
    -2.625,
    -3.11328125,
    -2.173828125,
    -2.357421875,
    -1.8974609375,
    -2.068359375,
    -1.6806640625,
    -1.400390625,
    -1.392578125,
    -1.9912109375,
    -1.9560546875,
    -0.493896484375,
    -1.76953125,
    -2.31640625,
    -1.2099609375,
    -1.2705078125,
    -1.58203125,
    -1.87109375,
    -1.37109375,
    -1.6484375,
    -0.96630859375,
    -1.0517578125,
    -1.6474609375,
    -1.205078125,
    -1.2919921875,
    -1.9794921875,
    -0.85400390625,
    -2.2578125,
    -2.458984375,
    -0.1263427734375,
    -1.681640625,
    -0.64990234375,
    -1.03125,
    -0.1162109375,
    -0.87255859375,
    -2.4765625,
    -0.7900390625,
    -1.251953125,
    -2.1328125,
    -1.3466796875,
    -1.6337890625,
    -1.513671875,
    -1.6259765625,
    -1.375,
    -0.80517578125,
    -0.0225067138671875,
    0.364990234375,
    -0.4169921875,
    0.9052734375,
    0.91943359375,
    -0.268310546875,
    -0.30615234375,
    -0.3427734375,
    -0.08544921875,
    -0.94384765625,
    0.267333984375,
    1.0244140625,
    1.388671875,
    -0.346435546875,
    -0.22021484375,
    -0.64990234375,
    0.609375,
    0.5751953125,
    -0.459716796875,
    0.166748046875,
    0.07275390625,
    0.4287109375,
    0.56640625,
    -0.54833984375,
    -0.63427734375,
    1.9140625,
    -0.364013671875,
    -0.57373046875,
    -0.333740234375,
    0.361328125,
    -0.712890625,
    -7.9453125
   ],
   "raw256_argmax": 249,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.44827014207839966,
    0.5515587329864502,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00017116019444074482
   ],
   "p": [
    0.4483468532562256,
    0.5516531467437744
   ],
   "p_oracle": [
    0.4691595137119293,
    0.5308405160903931
   ],
   "abstain": 0.00017116019444074482,
   "abstain_oracle": 0.00016973857418633997,
   "abs_delta_abstain": 1.4216202544048429e-06,
   "oracle_margin": 0.061681002378463745,
   "max_abs_delta_p": 0.020812660455703735,
   "mean_abs_delta_p": 0.02081264555454254,
   "max_abs_delta_raw_logits": 0.07616090774536133,
   "mean_abs_delta_raw_logits": 0.007669182610698044,
   "logits_min": -7.9453125,
   "logits_max": 1.9140625,
   "raw_logits_min": -7.9453125,
   "raw_logits_max": 1.9140625,
   "raw_logits_absmax": 7.9453125,
   "oracle_logits_min": -7.923232078552246,
   "oracle_logits_max": 1.9354100227355957,
   "oracle_raw_logits_absmax": 7.923232078552246,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4102001660503447,
   "seconds": 0.4102001660503447,
   "full_vocab_argmax_id": 249,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r06-phase",
   "request_id": "r06",
   "question_id": "phase",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 96,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.7919921875,
    3.208984375,
    -2.453125,
    -2.736328125,
    -2.013671875,
    -2.49609375,
    -2.369140625,
    -3.5625,
    -3.7421875,
    -2.44140625,
    -2.935546875,
    -3.861328125,
    -4.96875,
    -3.02734375,
    -4.15234375,
    -4.28125,
    -3.75390625,
    -4.24609375,
    -2.912109375,
    -2.8046875,
    -3.15625,
    -1.8251953125,
    -5.04296875,
    -6.75390625,
    -3.703125,
    -4.96875,
    -2.337890625,
    -5.11328125,
    -4.38671875,
    -6.484375,
    -4.45703125,
    -4.56640625,
    -6.03515625,
    -4.80859375,
    -3.072265625,
    -5.21484375,
    -5.1953125,
    -5.8359375,
    -4.1015625,
    -2.912109375,
    -6.8203125,
    -4.24609375,
    -4.73828125,
    -5.80859375,
    -4.140625,
    -4.8125,
    -4.375,
    -4.29296875,
    -5.1484375,
    -4.390625,
    -5.5859375,
    -4.83984375,
    -2.517578125,
    -4.87890625,
    -2.7734375,
    -3.33984375,
    -3.755859375,
    -4.59765625,
    -4.77734375,
    -5.73828125,
    -7.34375,
    -6.2421875,
    -7.6484375,
    -6.4453125,
    -3.3203125,
    -7.27734375,
    -7.375,
    -2.400390625,
    -7.484375,
    -3.826171875,
    -7.7421875,
    -5.09375,
    -2.513671875,
    -5.8359375,
    -3.7421875,
    -5.875,
    -3.955078125,
    -6.5234375,
    -4.1484375,
    -5.55078125,
    -4.296875,
    -4.0234375,
    -2.6015625,
    -4.15625,
    -3.39453125,
    -3.806640625,
    -2.46484375,
    -3.77734375,
    -2.9765625,
    -4.5703125,
    -3.265625,
    -3.951171875,
    -4.2421875,
    -3.90625,
    -4.3984375,
    -4.55859375,
    -5.046875,
    -3.7578125,
    -3.8046875,
    -4.1484375,
    -5.8203125,
    -4.37890625,
    -3.396484375,
    -4.01171875,
    -4.1796875,
    -5.96875,
    -4.1640625,
    -5.46484375,
    -3.9453125,
    -5.45703125,
    -4.23046875,
    -4.53515625,
    -4.984375,
    -6.37109375,
    -4.65234375,
    -3.92578125,
    -4.484375,
    -5.74609375,
    -4.1953125,
    -3.5078125,
    -3.35546875,
    -3.95703125,
    -3.1875,
    -3.767578125,
    -2.408203125,
    -5.625,
    -5.59765625,
    -3.0,
    -3.099609375,
    -5.29296875,
    -5.4921875,
    -3.353515625,
    -5.375,
    -5.8125,
    -4.7890625,
    -4.37890625,
    -3.875,
    -4.89453125,
    -5.67578125,
    -2.97265625,
    -3.52734375,
    -4.06640625,
    -3.591796875,
    -2.9453125,
    -4.75390625,
    -5.27734375,
    -5.8515625,
    -5.00390625,
    -4.60546875,
    -4.41796875,
    -4.71875,
    -3.09375,
    -0.95849609375,
    -3.36328125,
    -2.56640625,
    -1.583984375,
    -1.8916015625,
    -2.041015625,
    -3.400390625,
    -0.30615234375,
    -2.037109375,
    -2.001953125,
    -1.6806640625,
    -1.3447265625,
    -2.46484375,
    -2.220703125,
    -2.75,
    -1.8388671875,
    -3.337890625,
    -1.8505859375,
    -2.18359375,
    -2.4765625,
    -2.40234375,
    -2.966796875,
    -0.7041015625,
    -1.7509765625,
    -3.25390625,
    -3.212890625,
    -2.794921875,
    -3.140625,
    -3.353515625,
    -2.927734375,
    -2.458984375,
    -2.037109375,
    -2.416015625,
    -1.4033203125,
    -1.017578125,
    -1.5859375,
    -1.78515625,
    -1.90625,
    -0.74365234375,
    -1.62109375,
    -2.69921875,
    -1.6298828125,
    -1.4794921875,
    -2.01953125,
    -1.92578125,
    -1.7685546875,
    -1.4765625,
    -1.109375,
    -1.01171875,
    -1.7724609375,
    -1.7216796875,
    -1.798828125,
    -2.3125,
    -0.88427734375,
    -2.546875,
    -2.478515625,
    -0.459716796875,
    -2.1953125,
    -0.8173828125,
    -1.1904296875,
    -0.73828125,
    -1.349609375,
    -2.4453125,
    -1.2060546875,
    -1.6357421875,
    -2.15625,
    -1.6787109375,
    -1.7451171875,
    -1.3720703125,
    -1.6767578125,
    -1.359375,
    -1.1884765625,
    -0.2088623046875,
    -0.03070068359375,
    -0.46533203125,
    1.0048828125,
    0.96533203125,
    -0.01287078857421875,
    -0.79736328125,
    -0.481201171875,
    0.218994140625,
    -0.685546875,
    0.51025390625,
    1.19921875,
    1.453125,
    -0.59375,
    0.102294921875,
    -0.8935546875,
    0.371826171875,
    0.71630859375,
    -0.701171875,
    -0.1317138671875,
    -0.039215087890625,
    0.56640625,
    0.390380859375,
    -0.362548828125,
    -1.0029296875,
    1.5634765625,
    -0.06744384765625,
    -0.8408203125,
    -0.1612548828125,
    0.38916015625,
    -0.50732421875,
    -5.04296875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.00875281635671854,
    0.9861542582511902,
    0.004687040578573942,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00040582549991086125
   ],
   "p": [
    0.008756370283663273,
    0.9865546822547913,
    0.004688943736255169
   ],
   "p_oracle": [
    0.008610899560153484,
    0.9867118000984192,
    0.0046772826462984085
   ],
   "abstain": 0.00040582549991086125,
   "abstain_oracle": 0.0003977194719482213,
   "abs_delta_abstain": 8.106027962639928e-06,
   "oracle_margin": 0.9781008958816528,
   "max_abs_delta_p": 0.0001571178436279297,
   "mean_abs_delta_p": 0.00010474988569815953,
   "max_abs_delta_raw_logits": 0.03405117988586426,
   "mean_abs_delta_raw_logits": 0.00531766316271387,
   "logits_min": -7.7421875,
   "logits_max": 3.208984375,
   "raw_logits_min": -7.7421875,
   "raw_logits_max": 3.208984375,
   "raw_logits_absmax": 7.7421875,
   "oracle_logits_min": -7.740938186645508,
   "oracle_logits_max": 3.2222442626953125,
   "oracle_raw_logits_absmax": 7.740938186645508,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5377657500794157,
   "seconds": 0.5377657500794157,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r06-priority",
   "request_id": "r06",
   "question_id": "priority",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 68,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -3.197265625,
    -2.744140625,
    3.15234375,
    -1.5087890625,
    -2.61328125,
    -2.7734375,
    -2.388671875,
    -2.59375,
    -2.779296875,
    -1.7275390625,
    -2.884765625,
    -2.484375,
    -4.484375,
    -2.572265625,
    -3.390625,
    -4.1484375,
    -3.71484375,
    -4.43359375,
    -2.521484375,
    -2.4453125,
    -4.09765625,
    -1.4462890625,
    -5.2421875,
    -5.48828125,
    -3.396484375,
    -4.578125,
    -3.388671875,
    -4.1015625,
    -3.802734375,
    -5.74609375,
    -4.44921875,
    -5.0078125,
    -4.87890625,
    -4.078125,
    -2.75390625,
    -4.90625,
    -5.359375,
    -6.53125,
    -4.73046875,
    -3.21484375,
    -5.96875,
    -4.15625,
    -4.84375,
    -6.2578125,
    -4.3125,
    -4.88671875,
    -4.41796875,
    -4.3203125,
    -4.88671875,
    -4.75390625,
    -5.19921875,
    -4.77734375,
    -2.783203125,
    -4.84765625,
    -2.9296875,
    -3.640625,
    -4.09765625,
    -5.234375,
    -5.15234375,
    -6.078125,
    -7.4140625,
    -6.6328125,
    -7.4375,
    -5.91015625,
    -3.31640625,
    -7.08203125,
    -7.3515625,
    -2.951171875,
    -7.57421875,
    -4.5859375,
    -7.953125,
    -5.234375,
    -2.705078125,
    -5.515625,
    -2.79296875,
    -6.2734375,
    -3.8515625,
    -6.48046875,
    -4.2421875,
    -5.55078125,
    -4.453125,
    -3.8125,
    -2.453125,
    -4.53125,
    -3.314453125,
    -4.5859375,
    -2.583984375,
    -4.12890625,
    -3.021484375,
    -4.3125,
    -3.4296875,
    -3.744140625,
    -4.2421875,
    -3.24609375,
    -4.89453125,
    -4.62109375,
    -4.9609375,
    -3.375,
    -3.00390625,
    -4.58203125,
    -5.63671875,
    -4.52734375,
    -3.72265625,
    -3.73046875,
    -4.33984375,
    -5.4609375,
    -3.32421875,
    -5.140625,
    -4.0703125,
    -5.53515625,
    -3.9921875,
    -4.44140625,
    -4.9609375,
    -5.5859375,
    -4.47265625,
    -3.685546875,
    -4.3046875,
    -5.8125,
    -3.921875,
    -3.1328125,
    -3.3515625,
    -4.234375,
    -2.740234375,
    -3.076171875,
    -2.48046875,
    -5.11328125,
    -5.140625,
    -2.9140625,
    -2.6015625,
    -5.26171875,
    -5.359375,
    -2.421875,
    -5.53515625,
    -6.1640625,
    -5.53515625,
    -4.67578125,
    -3.65625,
    -4.37890625,
    -5.7109375,
    -3.025390625,
    -2.962890625,
    -4.0390625,
    -3.53515625,
    -2.783203125,
    -4.046875,
    -4.453125,
    -5.27734375,
    -5.12109375,
    -5.04296875,
    -4.41015625,
    -4.625,
    -2.79296875,
    -0.59326171875,
    -3.517578125,
    -2.201171875,
    -1.0634765625,
    -1.4580078125,
    -1.6953125,
    -2.78125,
    -0.1143798828125,
    -1.6572265625,
    -1.98828125,
    -1.255859375,
    -1.353515625,
    -3.13671875,
    -2.2734375,
    -3.0078125,
    -1.8603515625,
    -3.65234375,
    -1.8994140625,
    -2.017578125,
    -2.125,
    -1.9384765625,
    -2.595703125,
    -0.59375,
    -1.6181640625,
    -2.8046875,
    -3.197265625,
    -2.73046875,
    -2.806640625,
    -2.75,
    -2.435546875,
    -2.994140625,
    -2.052734375,
    -2.873046875,
    -1.640625,
    -2.177734375,
    -1.39453125,
    -1.484375,
    -2.578125,
    -0.63037109375,
    -1.6162109375,
    -2.521484375,
    -1.443359375,
    -1.8388671875,
    -2.30078125,
    -1.5537109375,
    -1.58984375,
    -1.626953125,
    -1.21875,
    -0.99853515625,
    -2.06640625,
    -1.6298828125,
    -1.755859375,
    -2.619140625,
    -0.75927734375,
    -2.234375,
    -2.44921875,
    0.0181427001953125,
    -1.7802734375,
    -0.9208984375,
    -1.03515625,
    -0.0631103515625,
    -1.4345703125,
    -3.029296875,
    -0.83984375,
    -1.4140625,
    -2.03515625,
    -1.2919921875,
    -1.55078125,
    -0.810546875,
    -1.8056640625,
    -1.283203125,
    -1.435546875,
    0.0784912109375,
    -0.0697021484375,
    -0.401611328125,
    1.0693359375,
    1.19921875,
    -0.1878662109375,
    -0.349853515625,
    -0.60693359375,
    -0.140869140625,
    -1.060546875,
    0.76123046875,
    0.837890625,
    1.212890625,
    -0.414306640625,
    -0.197021484375,
    -0.94873046875,
    0.6142578125,
    0.4677734375,
    -0.5830078125,
    0.1676025390625,
    -0.0947265625,
    1.0703125,
    0.446044921875,
    -0.712890625,
    -0.58740234375,
    1.7431640625,
    -0.302001953125,
    -0.55078125,
    -0.288330078125,
    0.39501953125,
    0.0406494140625,
    -6.09375
   ],
   "raw256_argmax": 2,
   "option_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "p_full": [
    0.002466570120304823,
    0.003784439293667674,
    0.9935890436172485,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00015985500067472458
   ],
   "p": [
    0.0024669645354151726,
    0.003785044653341174,
    0.9937479496002197
   ],
   "p_oracle": [
    0.002428818494081497,
    0.003745054127648473,
    0.993826150894165
   ],
   "abstain": 0.00015985500067472458,
   "abstain_oracle": 0.000158899201778695,
   "abs_delta_abstain": 9.557988960295916e-07,
   "oracle_margin": 0.9900810718536377,
   "max_abs_delta_p": 7.82012939453125e-05,
   "mean_abs_delta_p": 5.211262032389641e-05,
   "max_abs_delta_raw_logits": 0.026906490325927734,
   "mean_abs_delta_raw_logits": 0.005696268257452175,
   "logits_min": -7.953125,
   "logits_max": 3.15234375,
   "raw_logits_min": -7.953125,
   "raw_logits_max": 3.15234375,
   "raw_logits_absmax": 7.953125,
   "oracle_logits_min": -7.948171615600586,
   "oracle_logits_max": 3.1572537422180176,
   "oracle_raw_logits_absmax": 7.948171615600586,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.38229870900977403,
   "seconds": 0.38229870900977403,
   "full_vocab_argmax_id": 2,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r06-approved",
   "request_id": "r06",
   "question_id": "approved",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 90,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    3.3671875,
    -2.35546875,
    -2.21484375,
    -2.771484375,
    -2.369140625,
    -2.294921875,
    -2.55078125,
    -3.830078125,
    -3.14453125,
    -2.443359375,
    -2.73828125,
    -3.4375,
    -4.5390625,
    -2.60546875,
    -3.95703125,
    -4.11328125,
    -4.04296875,
    -4.59375,
    -2.70703125,
    -3.216796875,
    -3.251953125,
    -1.45703125,
    -4.98828125,
    -5.9921875,
    -3.568359375,
    -4.1953125,
    -2.796875,
    -4.9765625,
    -4.0234375,
    -6.46875,
    -4.0546875,
    -4.87890625,
    -6.09765625,
    -4.29296875,
    -2.998046875,
    -4.78125,
    -5.02734375,
    -6.3203125,
    -4.19140625,
    -3.552734375,
    -6.33984375,
    -4.40234375,
    -5.23046875,
    -6.1875,
    -4.00390625,
    -4.80078125,
    -4.1796875,
    -4.8828125,
    -5.26171875,
    -4.11328125,
    -5.375,
    -4.36328125,
    -3.259765625,
    -5.484375,
    -3.119140625,
    -3.443359375,
    -4.09765625,
    -4.76171875,
    -4.734375,
    -6.625,
    -7.3359375,
    -6.8046875,
    -7.92578125,
    -5.8515625,
    -3.876953125,
    -7.2109375,
    -7.61328125,
    -2.404296875,
    -7.34765625,
    -4.0859375,
    -8.078125,
    -5.30859375,
    -3.2578125,
    -5.484375,
    -3.69921875,
    -6.30859375,
    -4.10546875,
    -7.03125,
    -3.8984375,
    -5.9375,
    -4.8125,
    -4.1640625,
    -2.947265625,
    -4.52734375,
    -3.421875,
    -4.3984375,
    -2.392578125,
    -4.05859375,
    -3.1953125,
    -4.62109375,
    -3.609375,
    -4.234375,
    -4.31640625,
    -3.44140625,
    -4.50390625,
    -4.42578125,
    -4.9609375,
    -3.4453125,
    -3.078125,
    -4.13671875,
    -5.59765625,
    -4.7578125,
    -3.28125,
    -4.0703125,
    -4.58984375,
    -5.6640625,
    -3.869140625,
    -5.25390625,
    -4.4140625,
    -5.9765625,
    -4.62890625,
    -4.73046875,
    -5.109375,
    -6.01171875,
    -4.85546875,
    -3.87890625,
    -4.57421875,
    -5.82421875,
    -3.826171875,
    -3.830078125,
    -3.22265625,
    -4.33984375,
    -3.0625,
    -3.923828125,
    -2.2890625,
    -5.28515625,
    -5.765625,
    -2.71484375,
    -3.0546875,
    -5.58984375,
    -6.1484375,
    -2.787109375,
    -5.80078125,
    -6.4140625,
    -4.859375,
    -3.99609375,
    -3.677734375,
    -4.8359375,
    -6.02734375,
    -2.86328125,
    -3.521484375,
    -4.2421875,
    -3.421875,
    -2.416015625,
    -4.97265625,
    -5.14453125,
    -5.80859375,
    -5.47265625,
    -5.25390625,
    -4.5234375,
    -4.515625,
    -2.75390625,
    -1.0498046875,
    -3.68359375,
    -2.92578125,
    -1.607421875,
    -1.7900390625,
    -1.802734375,
    -2.830078125,
    -0.423583984375,
    -1.8505859375,
    -2.173828125,
    -1.66015625,
    -0.95654296875,
    -2.87109375,
    -1.798828125,
    -2.76171875,
    -2.466796875,
    -3.783203125,
    -1.8388671875,
    -2.1953125,
    -2.646484375,
    -2.36328125,
    -3.03125,
    -0.623046875,
    -1.98046875,
    -3.86328125,
    -3.279296875,
    -2.92578125,
    -2.185546875,
    -3.236328125,
    -2.5078125,
    -2.90234375,
    -2.10546875,
    -2.55078125,
    -1.7626953125,
    -1.3515625,
    -1.5517578125,
    -2.123046875,
    -2.6953125,
    -0.58154296875,
    -1.6669921875,
    -2.546875,
    -1.6064453125,
    -1.8408203125,
    -1.7451171875,
    -2.05078125,
    -1.8017578125,
    -2.029296875,
    -1.00390625,
    -1.2158203125,
    -1.9169921875,
    -1.4853515625,
    -1.931640625,
    -2.357421875,
    -1.041015625,
    -2.140625,
    -2.388671875,
    0.1573486328125,
    -1.9326171875,
    -1.3369140625,
    -1.0068359375,
    -0.00853729248046875,
    -0.626953125,
    -2.7421875,
    -0.98486328125,
    -1.705078125,
    -1.876953125,
    -1.154296875,
    -1.7822265625,
    -0.90576171875,
    -1.6025390625,
    -0.75,
    -1.1171875,
    -0.2568359375,
    -0.2374267578125,
    -0.56005859375,
    1.1025390625,
    0.814453125,
    -0.1002197265625,
    -0.47900390625,
    -0.58642578125,
    0.640625,
    -0.744140625,
    0.248291015625,
    0.438720703125,
    1.2099609375,
    -0.32470703125,
    0.2242431640625,
    -0.8330078125,
    0.386474609375,
    0.796875,
    -0.943359375,
    -0.513671875,
    -0.1800537109375,
    1.0205078125,
    0.60498046875,
    -0.73193359375,
    -0.58447265625,
    2.296875,
    -0.2919921875,
    -0.092041015625,
    -0.45947265625,
    0.206787109375,
    -0.34814453125,
    -4.49609375
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9962096214294434,
    0.003386398311704397,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0004039480409119278
   ],
   "p": [
    0.9966122508049011,
    0.0033877668902277946
   ],
   "p_oracle": [
    0.996588408946991,
    0.003411603160202503
   ],
   "abstain": 0.0004039480409119278,
   "abstain_oracle": 0.00040132785215973854,
   "abs_delta_abstain": 2.6201887521892786e-06,
   "oracle_margin": 0.9931768178939819,
   "max_abs_delta_p": 2.384185791015625e-05,
   "mean_abs_delta_p": 2.3839063942432404e-05,
   "max_abs_delta_raw_logits": 0.025831222534179688,
   "mean_abs_delta_raw_logits": 0.00587014970369637,
   "logits_min": -8.078125,
   "logits_max": 3.3671875,
   "raw_logits_min": -8.078125,
   "raw_logits_max": 3.3671875,
   "raw_logits_absmax": 8.078125,
   "oracle_logits_min": -8.074604034423828,
   "oracle_logits_max": 3.360363483428955,
   "oracle_raw_logits_absmax": 8.074604034423828,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5091823750408366,
   "seconds": 0.5091823750408366,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r07-team",
   "request_id": "r07",
   "question_id": "team",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 102,
   "nopts": 16,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.16015625,
    -1.826171875,
    -2.400390625,
    -1.9267578125,
    -2.990234375,
    -2.5390625,
    -2.654296875,
    -3.814453125,
    -3.66796875,
    -3.505859375,
    -1.9580078125,
    4.96875,
    -5.58984375,
    -4.71484375,
    -4.73828125,
    -4.8125,
    -5.20703125,
    -4.5078125,
    -3.10546875,
    -3.623046875,
    -3.9375,
    -1.484375,
    -6.1796875,
    -6.171875,
    -4.35546875,
    -5.625,
    -4.10546875,
    -5.171875,
    -4.78125,
    -7.5546875,
    -4.94921875,
    -6.87890625,
    -6.34375,
    -5.55078125,
    -3.671875,
    -5.66796875,
    -5.4140625,
    -5.58203125,
    -4.8984375,
    -2.98046875,
    -5.23046875,
    -4.6640625,
    -4.8984375,
    -4.72265625,
    -4.43359375,
    -5.0,
    -4.51171875,
    -4.453125,
    -5.1171875,
    -4.8046875,
    -4.6015625,
    -5.09375,
    -3.083984375,
    -5.02734375,
    -2.66796875,
    -2.330078125,
    -4.07421875,
    -4.1953125,
    -5.23828125,
    -5.6015625,
    -7.96875,
    -6.890625,
    -7.125,
    -5.91796875,
    -3.93359375,
    -6.98046875,
    -7.375,
    -2.30859375,
    -8.015625,
    -4.484375,
    -7.7109375,
    -5.30859375,
    -2.685546875,
    -5.81640625,
    -3.888671875,
    -6.26953125,
    -4.75390625,
    -7.19140625,
    -3.833984375,
    -5.75390625,
    -4.4921875,
    -4.2109375,
    -2.400390625,
    -4.26171875,
    -3.494140625,
    -4.578125,
    -2.09765625,
    -5.1484375,
    -2.888671875,
    -4.46484375,
    -3.953125,
    -3.943359375,
    -3.63671875,
    -4.25,
    -4.859375,
    -4.51171875,
    -5.2109375,
    -3.177734375,
    -3.697265625,
    -4.69921875,
    -5.98828125,
    -4.921875,
    -3.361328125,
    -4.19140625,
    -3.97265625,
    -6.03515625,
    -4.05859375,
    -5.96875,
    -4.234375,
    -6.0703125,
    -3.791015625,
    -5.0,
    -4.9453125,
    -6.328125,
    -5.4140625,
    -4.0859375,
    -4.234375,
    -6.3984375,
    -4.68359375,
    -3.3828125,
    -2.82421875,
    -3.73046875,
    -2.888671875,
    -3.47265625,
    -2.017578125,
    -5.88671875,
    -5.890625,
    -3.21484375,
    -3.38671875,
    -5.3046875,
    -5.32421875,
    -3.068359375,
    -5.57421875,
    -6.265625,
    -5.05078125,
    -4.953125,
    -3.625,
    -5.37109375,
    -6.109375,
    -3.380859375,
    -3.330078125,
    -4.0078125,
    -3.560546875,
    -2.591796875,
    -4.1484375,
    -5.046875,
    -5.21875,
    -5.14453125,
    -5.4375,
    -4.69140625,
    -5.171875,
    -2.744140625,
    -1.68359375,
    -3.064453125,
    -1.8994140625,
    -1.80859375,
    -0.95849609375,
    -1.728515625,
    -2.263671875,
    -0.55419921875,
    -1.76171875,
    -1.23828125,
    -1.8330078125,
    -1.130859375,
    -2.974609375,
    -2.173828125,
    -3.125,
    -2.189453125,
    -3.955078125,
    -2.056640625,
    -2.244140625,
    -2.458984375,
    -2.52734375,
    -2.919921875,
    -1.1591796875,
    -1.9599609375,
    -3.552734375,
    -3.662109375,
    -3.1484375,
    -3.326171875,
    -4.265625,
    -3.373046875,
    -3.0546875,
    -1.791015625,
    -2.806640625,
    -2.35546875,
    -1.814453125,
    -1.7421875,
    -2.59765625,
    -2.46875,
    -1.748046875,
    -1.80078125,
    -2.4296875,
    -1.4404296875,
    -1.2353515625,
    -2.1875,
    -2.20703125,
    -2.51171875,
    -1.611328125,
    -1.2744140625,
    -1.625,
    -2.560546875,
    -2.154296875,
    -2.427734375,
    -2.27734375,
    -1.650390625,
    -2.119140625,
    -2.228515625,
    0.326904296875,
    -2.412109375,
    -1.50390625,
    -1.8232421875,
    -0.348876953125,
    -1.5859375,
    -3.423828125,
    -1.04296875,
    -2.49609375,
    -2.490234375,
    -0.982421875,
    -2.40625,
    -1.4853515625,
    -1.314453125,
    -1.130859375,
    -1.3056640625,
    0.008819580078125,
    0.123046875,
    -0.406005859375,
    0.51953125,
    0.92236328125,
    -0.1873779296875,
    -1.0546875,
    -0.826171875,
    0.1268310546875,
    -0.466796875,
    0.6220703125,
    0.7587890625,
    0.94873046875,
    -0.841796875,
    0.2213134765625,
    -1.466796875,
    0.2117919921875,
    0.289306640625,
    -1.1689453125,
    0.37646484375,
    -0.376953125,
    0.82080078125,
    0.40771484375,
    0.267578125,
    0.01419830322265625,
    1.736328125,
    0.10906982421875,
    0.2587890625,
    -0.207275390625,
    0.71533203125,
    -0.2061767578125,
    -3.75390625
   ],
   "raw256_argmax": 11,
   "option_argmax": 11,
   "oracle_argmax": 11,
   "argmax_agrees": true,
   "p_full": [
    0.0030212481506168842,
    0.0016104005044326186,
    0.000936146592721343,
    0.00146441999822855,
    0.0005362202064134181,
    0.0008212016546167433,
    0.0007364978082478046,
    0.0002461406693328172,
    0.0002826719719450921,
    0.0003294527414254844,
    0.0014218194410204887,
    0.987983226776123,
    4.60020819446072e-05,
    0.0001051409708452411,
    0.00010283847950631753,
    9.587501699570566e-05,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00026063001132570207
   ],
   "p": [
    0.003022036049515009,
    0.0016108204144984484,
    0.0009363907156512141,
    0.0014648018404841423,
    0.0005363600212149322,
    0.000821415800601244,
    0.0007366898935288191,
    0.0002462048432789743,
    0.000282745691947639,
    0.0003295386559329927,
    0.0014221902238205075,
    0.9882408976554871,
    4.601407999871299e-05,
    0.00010516838665353134,
    0.0001028652986860834,
    9.5900017186068e-05
   ],
   "p_oracle": [
    0.0029451660811901093,
    0.001555267721414566,
    0.0009126935619860888,
    0.0014587680343538523,
    0.0005313093424774706,
    0.000829861091915518,
    0.0007131857564672828,
    0.00023690734815318137,
    0.00027930724900215864,
    0.00032101283431984484,
    0.0013732233783230186,
    0.9885022044181824,
    4.6371802454814315e-05,
    0.00010244426084682345,
    0.000100217446743045,
    9.20594684430398e-05
   ],
   "abstain": 0.00026063001132570207,
   "abstain_oracle": 0.00024898501578718424,
   "abs_delta_abstain": 1.1644995538517833e-05,
   "oracle_margin": 0.9855570197105408,
   "max_abs_delta_p": 0.0002613067626953125,
   "mean_abs_delta_p": 3.3766209071472986e-05,
   "max_abs_delta_raw_logits": 0.03812742233276367,
   "mean_abs_delta_raw_logits": 0.007116629421943799,
   "logits_min": -8.015625,
   "logits_max": 4.96875,
   "raw_logits_min": -8.015625,
   "raw_logits_max": 4.96875,
   "raw_logits_absmax": 8.015625,
   "oracle_logits_min": -8.008580207824707,
   "oracle_logits_max": 4.996329307556152,
   "oracle_raw_logits_absmax": 8.008580207824707,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5716223330236971,
   "seconds": 0.5716223330236971,
   "full_vocab_argmax_id": 11,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r07-condition",
   "request_id": "r07",
   "question_id": "condition",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 43,
   "nopts": 4,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.349609375,
    4.03125,
    -2.38671875,
    -3.09375,
    -2.28125,
    -2.951171875,
    -2.80078125,
    -3.759765625,
    -3.63671875,
    -2.734375,
    -3.384765625,
    -4.34765625,
    -5.625,
    -3.50390625,
    -4.41015625,
    -4.84375,
    -3.71875,
    -4.1484375,
    -2.927734375,
    -3.166015625,
    -3.361328125,
    -1.814453125,
    -5.1640625,
    -7.29296875,
    -3.990234375,
    -5.5703125,
    -2.537109375,
    -5.765625,
    -4.5,
    -7.13671875,
    -4.56640625,
    -5.07421875,
    -6.46484375,
    -5.03515625,
    -3.01953125,
    -5.46484375,
    -5.1875,
    -5.76171875,
    -4.1328125,
    -3.095703125,
    -7.5,
    -4.359375,
    -4.84765625,
    -6.15625,
    -4.71484375,
    -5.2734375,
    -4.6015625,
    -4.5390625,
    -5.3359375,
    -4.9296875,
    -6.015625,
    -4.8515625,
    -2.9296875,
    -5.19140625,
    -2.78125,
    -3.390625,
    -3.638671875,
    -4.55078125,
    -4.82421875,
    -5.9453125,
    -7.8046875,
    -6.6484375,
    -7.80859375,
    -6.83984375,
    -3.794921875,
    -7.57421875,
    -7.73046875,
    -2.326171875,
    -8.0078125,
    -4.2109375,
    -8.09375,
    -5.60546875,
    -2.7578125,
    -6.3359375,
    -3.734375,
    -6.578125,
    -4.31640625,
    -6.91796875,
    -4.40234375,
    -5.9609375,
    -4.328125,
    -4.3515625,
    -2.859375,
    -4.203125,
    -3.810546875,
    -3.93359375,
    -2.67578125,
    -3.986328125,
    -2.873046875,
    -5.23828125,
    -3.642578125,
    -4.2734375,
    -4.4140625,
    -4.30859375,
    -4.6953125,
    -4.91796875,
    -5.24609375,
    -3.931640625,
    -4.203125,
    -4.24609375,
    -6.41015625,
    -4.69140625,
    -3.74609375,
    -4.3828125,
    -4.6328125,
    -6.4296875,
    -4.63671875,
    -5.87109375,
    -4.4375,
    -5.6875,
    -4.47265625,
    -4.94140625,
    -5.44921875,
    -6.96875,
    -4.890625,
    -4.3125,
    -4.875,
    -6.2578125,
    -4.33984375,
    -3.826171875,
    -3.564453125,
    -4.0625,
    -3.34765625,
    -3.841796875,
    -2.60546875,
    -6.03125,
    -5.953125,
    -3.310546875,
    -3.115234375,
    -5.7265625,
    -5.93359375,
    -3.93359375,
    -5.75390625,
    -6.1015625,
    -5.2109375,
    -4.88671875,
    -4.125,
    -5.3359375,
    -5.96484375,
    -3.244140625,
    -3.8203125,
    -4.1953125,
    -3.943359375,
    -3.376953125,
    -5.09375,
    -5.55078125,
    -6.328125,
    -5.23828125,
    -4.76953125,
    -4.68359375,
    -4.86328125,
    -3.26171875,
    -1.169921875,
    -3.693359375,
    -2.8125,
    -1.681640625,
    -2.23828125,
    -2.203125,
    -3.5546875,
    -0.47998046875,
    -2.07421875,
    -2.072265625,
    -1.7353515625,
    -1.4462890625,
    -2.611328125,
    -2.513671875,
    -2.919921875,
    -1.7646484375,
    -3.3515625,
    -2.029296875,
    -2.353515625,
    -2.494140625,
    -2.638671875,
    -3.072265625,
    -0.93505859375,
    -2.078125,
    -3.30078125,
    -3.44140625,
    -3.185546875,
    -3.626953125,
    -3.4375,
    -3.453125,
    -2.822265625,
    -2.0625,
    -2.541015625,
    -1.583984375,
    -1.3671875,
    -1.94140625,
    -1.8623046875,
    -2.267578125,
    -1.0439453125,
    -1.8525390625,
    -3.095703125,
    -1.775390625,
    -1.5771484375,
    -2.072265625,
    -2.076171875,
    -2.029296875,
    -1.5908203125,
    -1.509765625,
    -1.2158203125,
    -2.078125,
    -1.916015625,
    -1.87890625,
    -2.52734375,
    -1.052734375,
    -2.81640625,
    -2.720703125,
    -0.39599609375,
    -2.435546875,
    -0.99462890625,
    -1.294921875,
    -0.84814453125,
    -1.5927734375,
    -2.375,
    -1.3232421875,
    -1.783203125,
    -2.28125,
    -1.828125,
    -1.9677734375,
    -1.22265625,
    -1.8740234375,
    -1.3857421875,
    -1.5205078125,
    -0.425537109375,
    0.2401123046875,
    -0.361328125,
    0.97021484375,
    0.98291015625,
    -0.0295562744140625,
    -0.9453125,
    -0.5439453125,
    0.1544189453125,
    -0.7724609375,
    0.59375,
    1.154296875,
    1.42578125,
    -0.642578125,
    0.049163818359375,
    -0.83740234375,
    0.357421875,
    0.77099609375,
    -0.95166015625,
    -0.291015625,
    0.209228515625,
    0.71240234375,
    0.351318359375,
    -0.3896484375,
    -0.9794921875,
    1.4150390625,
    0.1143798828125,
    -0.83447265625,
    -0.1363525390625,
    0.457275390625,
    -0.5498046875,
    -4.3515625
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0023951921612024307,
    0.9937449097633362,
    0.002312678610906005,
    0.0011858644429594278,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003613937587942928
   ],
   "p": [
    0.002396057825535536,
    0.9941040873527527,
    0.0023135144729167223,
    0.0011862930841743946
   ],
   "p_oracle": [
    0.0023597946856170893,
    0.9941675662994385,
    0.002299339510500431,
    0.0011732407147064805
   ],
   "abstain": 0.0003613937587942928,
   "abstain_oracle": 0.0003573091817088425,
   "abs_delta_abstain": 4.084577085450292e-06,
   "oracle_margin": 0.991807758808136,
   "max_abs_delta_p": 6.347894668579102e-05,
   "mean_abs_delta_p": 3.1742354622110724e-05,
   "max_abs_delta_raw_logits": 0.022991180419921875,
   "mean_abs_delta_raw_logits": 0.00488988880533725,
   "logits_min": -8.09375,
   "logits_max": 4.03125,
   "raw_logits_min": -8.09375,
   "raw_logits_max": 4.03125,
   "raw_logits_absmax": 8.09375,
   "oracle_logits_min": -8.100370407104492,
   "oracle_logits_max": 4.035501003265381,
   "oracle_raw_logits_absmax": 8.100370407104492,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.24857683293521404,
   "seconds": 0.24857683293521404,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r07-refund",
   "request_id": "r07",
   "question_id": "refund",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 47,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    3.333984375,
    -2.421875,
    -2.27734375,
    -2.830078125,
    -2.357421875,
    -2.6171875,
    -2.755859375,
    -4.0390625,
    -3.310546875,
    -2.556640625,
    -2.759765625,
    -3.3046875,
    -4.5390625,
    -2.64453125,
    -4.11328125,
    -4.3203125,
    -4.41796875,
    -4.56640625,
    -2.583984375,
    -3.390625,
    -3.453125,
    -1.4658203125,
    -4.93359375,
    -5.97265625,
    -3.8046875,
    -4.22265625,
    -3.037109375,
    -5.08984375,
    -4.13671875,
    -6.515625,
    -4.09765625,
    -5.03515625,
    -5.984375,
    -4.3046875,
    -2.958984375,
    -4.8984375,
    -5.1015625,
    -6.09765625,
    -4.20703125,
    -3.498046875,
    -6.359375,
    -4.13671875,
    -5.1875,
    -6.16796875,
    -4.078125,
    -4.87890625,
    -4.2421875,
    -4.85546875,
    -5.31640625,
    -4.07421875,
    -5.2890625,
    -4.265625,
    -3.3671875,
    -5.5078125,
    -3.064453125,
    -3.33984375,
    -4.14453125,
    -4.953125,
    -4.75,
    -6.6875,
    -7.390625,
    -6.984375,
    -8.03125,
    -5.84765625,
    -3.859375,
    -7.28515625,
    -7.78125,
    -2.517578125,
    -7.51171875,
    -4.20703125,
    -8.125,
    -5.3828125,
    -3.416015625,
    -5.48046875,
    -3.662109375,
    -6.46484375,
    -4.2578125,
    -7.16015625,
    -3.962890625,
    -6.1796875,
    -4.81640625,
    -4.16015625,
    -3.005859375,
    -4.671875,
    -3.6328125,
    -4.6015625,
    -2.474609375,
    -4.07421875,
    -3.240234375,
    -4.86328125,
    -3.736328125,
    -4.34375,
    -4.21875,
    -3.71875,
    -4.5546875,
    -4.59765625,
    -4.875,
    -3.51953125,
    -3.0703125,
    -4.2109375,
    -5.7109375,
    -4.94140625,
    -3.39453125,
    -3.9765625,
    -4.5703125,
    -5.78515625,
    -4.05859375,
    -5.36328125,
    -4.43359375,
    -6.1796875,
    -4.6640625,
    -4.7890625,
    -5.28125,
    -6.171875,
    -4.96875,
    -3.8984375,
    -4.6328125,
    -6.01171875,
    -4.0,
    -4.00390625,
    -3.451171875,
    -4.48046875,
    -3.1640625,
    -3.77734375,
    -2.50390625,
    -5.4609375,
    -5.82421875,
    -2.841796875,
    -3.041015625,
    -5.71484375,
    -6.23046875,
    -2.861328125,
    -5.91796875,
    -6.484375,
    -4.98828125,
    -3.951171875,
    -3.755859375,
    -4.85546875,
    -6.15625,
    -2.904296875,
    -3.521484375,
    -4.390625,
    -3.48828125,
    -2.45703125,
    -5.04296875,
    -5.19921875,
    -5.9296875,
    -5.75,
    -5.2578125,
    -4.68359375,
    -4.66015625,
    -2.78125,
    -1.103515625,
    -3.744140625,
    -3.041015625,
    -1.578125,
    -1.693359375,
    -1.9248046875,
    -2.890625,
    -0.52294921875,
    -1.87109375,
    -2.142578125,
    -1.7138671875,
    -0.82958984375,
    -2.916015625,
    -1.8369140625,
    -2.72265625,
    -2.560546875,
    -3.880859375,
    -1.939453125,
    -2.279296875,
    -2.62109375,
    -2.271484375,
    -3.05859375,
    -0.7529296875,
    -2.166015625,
    -3.9140625,
    -3.412109375,
    -2.830078125,
    -2.298828125,
    -3.369140625,
    -2.6640625,
    -3.119140625,
    -2.130859375,
    -2.546875,
    -1.73046875,
    -1.478515625,
    -1.640625,
    -2.208984375,
    -2.802734375,
    -0.6630859375,
    -1.775390625,
    -2.591796875,
    -1.556640625,
    -1.8798828125,
    -1.841796875,
    -2.275390625,
    -1.9521484375,
    -2.0859375,
    -1.3662109375,
    -1.4287109375,
    -2.044921875,
    -1.6884765625,
    -1.974609375,
    -2.34375,
    -1.0908203125,
    -2.28125,
    -2.421875,
    0.2119140625,
    -2.005859375,
    -1.46875,
    -1.0546875,
    -0.064453125,
    -0.6220703125,
    -2.751953125,
    -0.98779296875,
    -1.66796875,
    -1.9833984375,
    -1.154296875,
    -1.7421875,
    -1.029296875,
    -1.5546875,
    -0.60400390625,
    -1.2509765625,
    -0.323486328125,
    -0.1661376953125,
    -0.6103515625,
    0.919921875,
    0.84033203125,
    -0.2177734375,
    -0.460205078125,
    -0.6708984375,
    0.5458984375,
    -0.69384765625,
    0.40625,
    0.34521484375,
    1.1796875,
    -0.31640625,
    0.207763671875,
    -0.90673828125,
    0.488525390625,
    0.72119140625,
    -0.94580078125,
    -0.5537109375,
    -0.10302734375,
    1.001953125,
    0.50927734375,
    -0.87109375,
    -0.67578125,
    2.3125,
    -0.103759765625,
    -0.0648193359375,
    -0.333251953125,
    0.357421875,
    -0.43212890625,
    -4.66796875
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9963709115982056,
    0.0032770673278719187,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00035202704020775855
   ],
   "p": [
    0.9967218041419983,
    0.0032782212365418673
   ],
   "p_oracle": [
    0.9967036843299866,
    0.0032962816767394543
   ],
   "abstain": 0.00035202704020775855,
   "abstain_oracle": 0.0003536264121066779,
   "abs_delta_abstain": 1.599371898919344e-06,
   "oracle_margin": 0.9934074282646179,
   "max_abs_delta_p": 1.811981201171875e-05,
   "mean_abs_delta_p": 1.809012610465288e-05,
   "max_abs_delta_raw_logits": 0.026428937911987305,
   "mean_abs_delta_raw_logits": 0.005466494301799685,
   "logits_min": -8.125,
   "logits_max": 3.333984375,
   "raw_logits_min": -8.125,
   "raw_logits_max": 3.333984375,
   "raw_logits_absmax": 8.125,
   "oracle_logits_min": -8.115945816040039,
   "oracle_logits_max": 3.3308520317077637,
   "oracle_raw_logits_absmax": 8.115945816040039,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2687187090050429,
   "seconds": 0.2687187090050429,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r08-bin",
   "request_id": "r08",
   "question_id": "bin",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 65,
   "nopts": 12,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.62109375,
    -1.9765625,
    -2.673828125,
    -2.146484375,
    -1.8447265625,
    -1.076171875,
    6.4375,
    -5.390625,
    -2.373046875,
    -2.501953125,
    -3.755859375,
    -4.4375,
    -4.953125,
    -2.025390625,
    -3.98046875,
    -5.21484375,
    -5.0,
    -4.8515625,
    -2.2265625,
    -2.609375,
    -3.638671875,
    -1.67578125,
    -5.38671875,
    -6.41796875,
    -4.71484375,
    -4.90234375,
    -2.37890625,
    -5.23828125,
    -4.71484375,
    -6.83203125,
    -5.296875,
    -5.07421875,
    -6.40625,
    -4.6796875,
    -2.86328125,
    -4.796875,
    -5.39453125,
    -6.60546875,
    -3.322265625,
    -4.01953125,
    -6.515625,
    -4.1328125,
    -4.875,
    -5.36328125,
    -4.64453125,
    -4.5234375,
    -5.44140625,
    -5.5,
    -5.43359375,
    -3.30078125,
    -5.73828125,
    -4.15234375,
    -1.546875,
    -5.69921875,
    -2.708984375,
    -3.580078125,
    -4.5078125,
    -5.4375,
    -4.7734375,
    -6.57421875,
    -7.73046875,
    -6.4921875,
    -7.61328125,
    -6.19921875,
    -3.771484375,
    -7.44140625,
    -7.26171875,
    -2.958984375,
    -7.6796875,
    -3.791015625,
    -7.68359375,
    -4.86328125,
    -2.84375,
    -6.1640625,
    -3.5625,
    -5.87890625,
    -4.01171875,
    -6.796875,
    -3.748046875,
    -5.57421875,
    -4.87890625,
    -4.3125,
    -2.439453125,
    -4.140625,
    -3.126953125,
    -4.51953125,
    -2.7734375,
    -4.04296875,
    -2.869140625,
    -4.52734375,
    -3.7265625,
    -4.93359375,
    -4.82421875,
    -3.974609375,
    -5.37890625,
    -4.93359375,
    -5.27734375,
    -4.5234375,
    -4.1640625,
    -4.23828125,
    -5.65234375,
    -4.6953125,
    -3.66015625,
    -4.3984375,
    -4.2734375,
    -5.51171875,
    -3.83203125,
    -5.40234375,
    -4.578125,
    -5.56640625,
    -3.73046875,
    -4.51953125,
    -5.18359375,
    -6.9453125,
    -5.078125,
    -4.41015625,
    -4.38671875,
    -5.37109375,
    -3.919921875,
    -3.197265625,
    -3.05078125,
    -4.59765625,
    -3.41015625,
    -4.24609375,
    -2.904296875,
    -5.76171875,
    -5.82421875,
    -2.693359375,
    -3.0859375,
    -6.1953125,
    -6.24609375,
    -2.833984375,
    -5.7578125,
    -6.52734375,
    -5.546875,
    -4.12890625,
    -3.958984375,
    -4.05078125,
    -6.25,
    -3.296875,
    -3.77734375,
    -3.578125,
    -4.25390625,
    -2.8046875,
    -4.7421875,
    -5.53125,
    -5.82421875,
    -5.3515625,
    -5.55078125,
    -4.08203125,
    -5.52734375,
    -3.6875,
    -0.83447265625,
    -3.796875,
    -2.525390625,
    -1.763671875,
    -1.892578125,
    -2.404296875,
    -3.693359375,
    -0.19482421875,
    -1.4404296875,
    -2.46484375,
    -1.671875,
    -0.896484375,
    -4.21484375,
    -2.37109375,
    -2.607421875,
    -3.181640625,
    -3.021484375,
    -1.671875,
    -2.359375,
    -2.443359375,
    -2.244140625,
    -2.357421875,
    -1.03125,
    -2.220703125,
    -3.7578125,
    -3.390625,
    -3.306640625,
    -2.94140625,
    -2.943359375,
    -2.880859375,
    -2.849609375,
    -2.271484375,
    -3.146484375,
    -2.298828125,
    -2.279296875,
    -1.595703125,
    -1.9541015625,
    -2.37890625,
    -0.87353515625,
    -1.6416015625,
    -2.677734375,
    -2.05859375,
    -2.44140625,
    -1.994140625,
    -2.609375,
    -1.55859375,
    -2.056640625,
    -1.3515625,
    -1.564453125,
    -2.24609375,
    -2.7109375,
    -1.9990234375,
    -2.208984375,
    -1.1748046875,
    -2.337890625,
    -1.9794921875,
    0.3955078125,
    -2.482421875,
    -0.71435546875,
    -0.421630859375,
    -1.1162109375,
    -1.33203125,
    -2.982421875,
    -0.82666015625,
    -2.4140625,
    -2.4375,
    -1.32421875,
    -1.4853515625,
    -0.740234375,
    -1.822265625,
    -1.3603515625,
    -1.2041015625,
    -0.381103515625,
    0.05120849609375,
    -0.9072265625,
    1.1845703125,
    0.748046875,
    -0.0074310302734375,
    -0.74365234375,
    -0.4384765625,
    0.160400390625,
    -0.8115234375,
    0.70654296875,
    1.1943359375,
    1.7529296875,
    -0.90771484375,
    -0.3232421875,
    -0.9541015625,
    1.138671875,
    0.455078125,
    -0.869140625,
    -0.35205078125,
    -0.4794921875,
    0.7861328125,
    0.031280517578125,
    -0.916015625,
    -1.06640625,
    1.2060546875,
    0.198486328125,
    -0.396484375,
    0.2100830078125,
    -0.017547607421875,
    -0.205810546875,
    -4.28125
   ],
   "raw256_argmax": 6,
   "option_argmax": 6,
   "oracle_argmax": 6,
   "argmax_agrees": true,
   "p_full": [
    0.0004924381501041353,
    0.00035197383840568364,
    0.00018215322052128613,
    0.00029977448866702616,
    0.00039865737198852,
    0.0008239900344051421,
    0.9968408346176147,
    1.3989146282256115e-05,
    0.00024201421183533967,
    0.00021426613966468722,
    6.553927232744172e-05,
    3.442217712290585e-05,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    3.989741890109144e-05
   ],
   "p": [
    0.0004924577660858631,
    0.0003519878664519638,
    0.00018216048192698509,
    0.0002997864503413439,
    0.0003986732626799494,
    0.0008240229217335582,
    0.9968805909156799,
    1.3989703802508302e-05,
    0.00024202385975513607,
    0.00021427468163892627,
    6.554188439622521e-05,
    3.4423548640916124e-05
   ],
   "p_oracle": [
    0.0004889668780378997,
    0.00035156647209078074,
    0.00018288858700543642,
    0.0002941089041996747,
    0.0003917153226211667,
    0.0008335465681739151,
    0.9968871474266052,
    1.3746554941462819e-05,
    0.0002432497713016346,
    0.00021473021479323506,
    6.454402318922803e-05,
    3.38076388288755e-05
   ],
   "abstain": 3.989741890109144e-05,
   "abstain_oracle": 3.9258102333405986e-05,
   "abs_delta_abstain": 6.393165676854551e-07,
   "oracle_margin": 0.9960535764694214,
   "max_abs_delta_p": 9.52364644035697e-06,
   "mean_abs_delta_p": 3.074532969549182e-06,
   "max_abs_delta_raw_logits": 0.022936105728149414,
   "mean_abs_delta_raw_logits": 0.00624737111502327,
   "logits_min": -7.73046875,
   "logits_max": 6.4375,
   "raw_logits_min": -7.73046875,
   "raw_logits_max": 6.4375,
   "raw_logits_absmax": 7.73046875,
   "oracle_logits_min": -7.736870288848877,
   "oracle_logits_max": 6.4470014572143555,
   "oracle_raw_logits_absmax": 7.736870288848877,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.36763041594531387,
   "seconds": 0.36763041594531387,
   "full_vocab_argmax_id": 6,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r08-lamp",
   "request_id": "r08",
   "question_id": "lamp",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 54,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.37890625,
    3.6953125,
    -3.12109375,
    -2.677734375,
    -2.109375,
    -2.671875,
    -2.748046875,
    -3.90234375,
    -3.73046875,
    -2.966796875,
    -3.421875,
    -4.296875,
    -5.8515625,
    -3.580078125,
    -4.60546875,
    -4.921875,
    -3.892578125,
    -4.16796875,
    -3.123046875,
    -3.26953125,
    -3.453125,
    -1.7919921875,
    -5.07421875,
    -7.38671875,
    -4.0625,
    -5.37890625,
    -2.548828125,
    -5.88671875,
    -4.5625,
    -7.25390625,
    -4.76171875,
    -5.109375,
    -6.67578125,
    -5.33984375,
    -3.189453125,
    -5.7734375,
    -5.3125,
    -6.05078125,
    -4.14453125,
    -3.224609375,
    -7.484375,
    -4.375,
    -4.79296875,
    -6.06640625,
    -4.73046875,
    -5.15234375,
    -4.7421875,
    -4.70703125,
    -5.1875,
    -4.78125,
    -5.984375,
    -4.86328125,
    -2.73828125,
    -5.30859375,
    -2.865234375,
    -3.16796875,
    -3.638671875,
    -4.66796875,
    -4.90625,
    -6.18359375,
    -7.77734375,
    -6.76953125,
    -7.88671875,
    -6.8046875,
    -3.724609375,
    -7.7578125,
    -7.84375,
    -2.32421875,
    -8.03125,
    -4.078125,
    -8.3203125,
    -5.71875,
    -2.869140625,
    -6.35546875,
    -3.94921875,
    -6.65234375,
    -4.44921875,
    -7.01171875,
    -4.19921875,
    -6.0,
    -4.35546875,
    -4.41796875,
    -2.849609375,
    -4.27734375,
    -3.70703125,
    -3.939453125,
    -2.748046875,
    -3.9453125,
    -2.8828125,
    -5.203125,
    -3.703125,
    -4.28125,
    -4.56640625,
    -4.18359375,
    -4.7265625,
    -4.94140625,
    -5.3671875,
    -4.140625,
    -4.32421875,
    -4.3828125,
    -6.50390625,
    -4.94140625,
    -3.703125,
    -4.390625,
    -4.6953125,
    -6.48046875,
    -4.68359375,
    -5.90625,
    -4.5234375,
    -5.7109375,
    -4.35546875,
    -5.0234375,
    -5.5078125,
    -7.046875,
    -4.8828125,
    -4.4453125,
    -4.81640625,
    -6.296875,
    -4.38671875,
    -3.6875,
    -3.5859375,
    -3.96484375,
    -3.376953125,
    -4.0078125,
    -2.638671875,
    -6.21484375,
    -6.09765625,
    -3.3046875,
    -3.275390625,
    -5.8203125,
    -5.95703125,
    -4.09375,
    -5.80078125,
    -6.2421875,
    -5.23046875,
    -4.92578125,
    -4.27734375,
    -5.15234375,
    -6.125,
    -3.23046875,
    -3.947265625,
    -4.2578125,
    -4.203125,
    -3.3125,
    -5.2578125,
    -5.6328125,
    -6.41796875,
    -5.40234375,
    -4.93359375,
    -4.75,
    -5.03125,
    -3.474609375,
    -1.076171875,
    -3.712890625,
    -2.935546875,
    -1.75390625,
    -2.25390625,
    -2.1796875,
    -3.572265625,
    -0.436767578125,
    -2.20703125,
    -2.109375,
    -1.716796875,
    -1.2939453125,
    -2.712890625,
    -2.62890625,
    -2.908203125,
    -1.97265625,
    -3.3984375,
    -1.9833984375,
    -2.423828125,
    -2.630859375,
    -2.751953125,
    -3.1484375,
    -1.041015625,
    -1.955078125,
    -3.33203125,
    -3.3203125,
    -3.177734375,
    -3.587890625,
    -3.56640625,
    -3.33984375,
    -2.91015625,
    -2.12109375,
    -2.607421875,
    -1.50390625,
    -1.2900390625,
    -1.8916015625,
    -1.8134765625,
    -2.23046875,
    -1.0458984375,
    -1.7041015625,
    -3.119140625,
    -1.85546875,
    -1.513671875,
    -2.099609375,
    -2.224609375,
    -2.083984375,
    -1.818359375,
    -1.3759765625,
    -1.171875,
    -2.021484375,
    -1.8681640625,
    -1.9248046875,
    -2.42578125,
    -1.15625,
    -2.8515625,
    -2.67578125,
    -0.372314453125,
    -2.57421875,
    -0.82861328125,
    -1.2890625,
    -0.82177734375,
    -1.6220703125,
    -2.41796875,
    -1.3369140625,
    -1.92578125,
    -2.400390625,
    -1.865234375,
    -2.02734375,
    -1.28515625,
    -1.693359375,
    -1.3818359375,
    -1.560546875,
    -0.2470703125,
    0.146484375,
    -0.266845703125,
    1.0625,
    0.94970703125,
    -0.06109619140625,
    -0.95849609375,
    -0.63134765625,
    0.27978515625,
    -0.79345703125,
    0.5869140625,
    1.0615234375,
    1.4833984375,
    -0.59521484375,
    0.08123779296875,
    -0.740234375,
    0.321533203125,
    0.7939453125,
    -0.7197265625,
    -0.382080078125,
    0.0673828125,
    0.7412109375,
    0.3134765625,
    -0.309814453125,
    -0.9443359375,
    1.5830078125,
    0.10809326171875,
    -0.69287109375,
    -0.187744140625,
    0.61279296875,
    -0.4296875,
    -4.296875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0032030176371335983,
    0.9946850538253784,
    0.0015887487679719925,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0005231865798123181
   ],
   "p": [
    0.00320469425059855,
    0.9952057003974915,
    0.0015895803226158023
   ],
   "p_oracle": [
    0.0031704173889011145,
    0.9952751398086548,
    0.0015544575871899724
   ],
   "abstain": 0.0005231865798123181,
   "abstain_oracle": 0.0005142955924384296,
   "abs_delta_abstain": 8.890987373888493e-06,
   "oracle_margin": 0.992104709148407,
   "max_abs_delta_p": 6.943941116333008e-05,
   "mean_abs_delta_p": 4.627966942886511e-05,
   "max_abs_delta_raw_logits": 0.027164459228515625,
   "mean_abs_delta_raw_logits": 0.0049889210786204785,
   "logits_min": -8.3203125,
   "logits_max": 3.6953125,
   "raw_logits_min": -8.3203125,
   "raw_logits_max": 3.6953125,
   "raw_logits_absmax": 8.3203125,
   "oracle_logits_min": -8.319535255432129,
   "oracle_logits_max": 3.7114009857177734,
   "oracle_raw_logits_absmax": 8.319535255432129,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3097347499569878,
   "seconds": 0.3097347499569878,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r08-closed",
   "request_id": "r08",
   "question_id": "closed",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 36,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -3.4765625,
    1.4052734375,
    -1.35546875,
    -1.4912109375,
    -1.09375,
    -2.572265625,
    -1.7099609375,
    -2.751953125,
    -3.0859375,
    -1.5830078125,
    -2.32421875,
    -3.6875,
    -4.44140625,
    -2.177734375,
    -2.994140625,
    -4.2109375,
    -3.806640625,
    -4.59765625,
    -2.6640625,
    -2.4140625,
    -3.37890625,
    -1.5693359375,
    -4.9609375,
    -6.484375,
    -3.822265625,
    -4.5703125,
    -2.55859375,
    -4.91015625,
    -3.78515625,
    -5.6171875,
    -4.2109375,
    -3.857421875,
    -5.7421875,
    -4.33984375,
    -2.419921875,
    -4.8515625,
    -5.01953125,
    -5.59765625,
    -4.0078125,
    -2.9765625,
    -6.8828125,
    -4.0234375,
    -4.50390625,
    -5.90625,
    -3.58984375,
    -4.54296875,
    -4.25390625,
    -4.4453125,
    -4.83984375,
    -4.1484375,
    -5.26953125,
    -4.3203125,
    -2.560546875,
    -4.69921875,
    -2.396484375,
    -3.212890625,
    -3.3515625,
    -4.5546875,
    -4.2890625,
    -5.11328125,
    -6.96875,
    -6.0,
    -7.203125,
    -6.07421875,
    -3.095703125,
    -7.1796875,
    -7.234375,
    -2.2734375,
    -7.51171875,
    -4.13671875,
    -7.546875,
    -5.08203125,
    -2.25,
    -5.59765625,
    -3.60546875,
    -5.796875,
    -4.11328125,
    -6.23046875,
    -3.82421875,
    -5.484375,
    -4.12890625,
    -3.76953125,
    -2.546875,
    -3.830078125,
    -3.09765625,
    -3.9921875,
    -2.408203125,
    -3.30859375,
    -2.76953125,
    -4.3515625,
    -3.279296875,
    -3.869140625,
    -3.4765625,
    -4.0,
    -4.5,
    -4.72265625,
    -4.69140625,
    -3.591796875,
    -3.537109375,
    -4.0546875,
    -5.30859375,
    -4.5390625,
    -3.30078125,
    -3.57421875,
    -4.3125,
    -5.48046875,
    -3.576171875,
    -5.1484375,
    -3.921875,
    -4.984375,
    -4.21484375,
    -4.296875,
    -4.5390625,
    -6.00390625,
    -4.34375,
    -3.61328125,
    -4.421875,
    -5.421875,
    -4.11328125,
    -3.09765625,
    -3.16015625,
    -3.9453125,
    -3.125,
    -3.525390625,
    -2.44921875,
    -5.41015625,
    -4.9296875,
    -2.87109375,
    -2.919921875,
    -5.00390625,
    -5.390625,
    -2.87890625,
    -5.46875,
    -5.734375,
    -4.91015625,
    -4.23828125,
    -3.525390625,
    -4.4921875,
    -5.53125,
    -2.419921875,
    -3.27734375,
    -3.904296875,
    -3.5546875,
    -3.310546875,
    -4.37890625,
    -4.66796875,
    -5.3828125,
    -4.81640625,
    -4.890625,
    -4.09375,
    -4.46484375,
    -3.041015625,
    -0.84814453125,
    -3.033203125,
    -2.291015625,
    -1.31640625,
    -1.6357421875,
    -1.876953125,
    -3.1484375,
    -0.454833984375,
    -1.9560546875,
    -1.92578125,
    -1.2255859375,
    -1.181640625,
    -2.638671875,
    -2.25,
    -2.44140625,
    -1.48046875,
    -3.142578125,
    -1.88671875,
    -1.6728515625,
    -2.32421875,
    -1.9599609375,
    -2.609375,
    -0.74072265625,
    -1.580078125,
    -2.84765625,
    -3.16796875,
    -2.60546875,
    -3.111328125,
    -2.865234375,
    -2.5625,
    -2.697265625,
    -1.748046875,
    -2.19921875,
    -1.3310546875,
    -1.447265625,
    -1.51171875,
    -1.4541015625,
    -1.876953125,
    -0.51953125,
    -1.4365234375,
    -2.478515625,
    -1.3388671875,
    -1.408203125,
    -1.87109375,
    -1.859375,
    -1.4794921875,
    -1.318359375,
    -1.048828125,
    -1.20703125,
    -1.484375,
    -1.38671875,
    -1.4560546875,
    -2.037109375,
    -0.5888671875,
    -2.392578125,
    -2.22265625,
    -0.05914306640625,
    -1.7734375,
    -0.533203125,
    -0.966796875,
    -0.517578125,
    -0.81884765625,
    -2.296875,
    -0.8583984375,
    -1.5595703125,
    -2.142578125,
    -1.5888671875,
    -1.537109375,
    -1.255859375,
    -1.5703125,
    -1.46875,
    -1.25,
    -0.1971435546875,
    0.329833984375,
    -0.219482421875,
    1.0888671875,
    0.99072265625,
    -0.019012451171875,
    -0.7109375,
    -0.45751953125,
    -0.07427978515625,
    -0.77587890625,
    0.53369140625,
    1.2763671875,
    1.66796875,
    -0.783203125,
    -0.2366943359375,
    -0.759765625,
    0.81396484375,
    0.7802734375,
    -0.552734375,
    -0.13134765625,
    -0.08380126953125,
    0.5263671875,
    0.2435302734375,
    -0.68212890625,
    -1.009765625,
    1.4833984375,
    0.01126861572265625,
    -0.89111328125,
    -0.07421875,
    0.62060546875,
    -0.56103515625,
    -6.66015625
   ],
   "raw256_argmax": 236,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.007772553712129593,
    0.9918984174728394,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003290335589554161
   ],
   "p": [
    0.007775112055242062,
    0.9922248721122742
   ],
   "p_oracle": [
    0.007764671463519335,
    0.9922353625297546
   ],
   "abstain": 0.0003290335589554161,
   "abstain_oracle": 0.0003250511654186994,
   "abs_delta_abstain": 3.9823935367167e-06,
   "oracle_margin": 0.9844706654548645,
   "max_abs_delta_p": 1.049041748046875e-05,
   "mean_abs_delta_p": 1.0465504601597786e-05,
   "max_abs_delta_raw_logits": 0.018729209899902344,
   "mean_abs_delta_raw_logits": 0.005318108247593045,
   "logits_min": -7.546875,
   "logits_max": 1.66796875,
   "raw_logits_min": -7.546875,
   "raw_logits_max": 1.66796875,
   "raw_logits_absmax": 7.546875,
   "oracle_logits_min": -7.544649600982666,
   "oracle_logits_max": 1.6727880239486694,
   "oracle_raw_logits_absmax": 7.544649600982666,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.20903387502767146,
   "seconds": 0.20903387502767146,
   "full_vocab_argmax_id": 236,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r09-liquid",
   "request_id": "r09",
   "question_id": "liquid",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 46,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -3.498046875,
    2.640625,
    -2.69140625,
    -3.06640625,
    -2.939453125,
    -3.392578125,
    -3.06640625,
    -3.703125,
    -3.94140625,
    -2.861328125,
    -3.421875,
    -3.830078125,
    -5.0703125,
    -3.017578125,
    -4.1875,
    -4.4296875,
    -4.01953125,
    -4.08203125,
    -3.056640625,
    -3.326171875,
    -3.69140625,
    -2.369140625,
    -5.46875,
    -6.93359375,
    -3.92578125,
    -5.62109375,
    -2.5546875,
    -5.26171875,
    -4.72265625,
    -6.6171875,
    -4.765625,
    -4.5859375,
    -6.328125,
    -4.94921875,
    -3.583984375,
    -5.5390625,
    -5.671875,
    -5.9921875,
    -4.42578125,
    -3.17578125,
    -7.23046875,
    -4.890625,
    -4.7734375,
    -5.9375,
    -4.6328125,
    -5.3515625,
    -4.50390625,
    -4.2109375,
    -5.65234375,
    -4.734375,
    -5.68359375,
    -4.99609375,
    -2.978515625,
    -4.7890625,
    -2.9921875,
    -3.51953125,
    -3.572265625,
    -4.09765625,
    -4.90625,
    -5.15234375,
    -7.53515625,
    -6.39453125,
    -7.671875,
    -6.765625,
    -3.716796875,
    -7.37890625,
    -7.75390625,
    -2.60546875,
    -8.109375,
    -4.44921875,
    -7.71875,
    -5.16796875,
    -2.3046875,
    -6.23828125,
    -3.88671875,
    -6.2109375,
    -4.296875,
    -6.71484375,
    -4.34375,
    -5.70703125,
    -4.390625,
    -4.05859375,
    -2.791015625,
    -4.44921875,
    -3.638671875,
    -4.0390625,
    -2.546875,
    -3.814453125,
    -3.275390625,
    -4.875,
    -3.802734375,
    -4.18359375,
    -4.4453125,
    -4.328125,
    -4.55078125,
    -4.61328125,
    -5.234375,
    -3.904296875,
    -4.17578125,
    -4.25,
    -6.2421875,
    -4.30859375,
    -3.93359375,
    -4.33984375,
    -4.55078125,
    -6.19140625,
    -4.34375,
    -5.76953125,
    -3.958984375,
    -5.75,
    -4.41796875,
    -4.76953125,
    -5.28515625,
    -6.3515625,
    -5.02734375,
    -3.962890625,
    -4.7578125,
    -6.453125,
    -4.75390625,
    -3.521484375,
    -3.33984375,
    -4.03125,
    -3.328125,
    -3.830078125,
    -2.455078125,
    -5.98828125,
    -5.94140625,
    -3.208984375,
    -3.37109375,
    -5.640625,
    -5.4921875,
    -3.99609375,
    -5.69921875,
    -6.13671875,
    -4.875,
    -5.1015625,
    -3.982421875,
    -4.90625,
    -6.09765625,
    -3.07421875,
    -3.7109375,
    -4.1953125,
    -3.58984375,
    -3.595703125,
    -4.5234375,
    -5.1328125,
    -6.2734375,
    -5.23046875,
    -4.578125,
    -4.26171875,
    -4.9375,
    -3.2890625,
    -1.462890625,
    -3.623046875,
    -2.263671875,
    -1.6845703125,
    -2.0078125,
    -2.001953125,
    -3.376953125,
    -0.7353515625,
    -2.345703125,
    -2.041015625,
    -1.912109375,
    -1.5849609375,
    -2.537109375,
    -2.337890625,
    -3.05859375,
    -1.8662109375,
    -3.6171875,
    -2.34765625,
    -2.4453125,
    -2.5546875,
    -2.31640625,
    -3.072265625,
    -0.962890625,
    -2.1484375,
    -3.32421875,
    -3.5546875,
    -3.52734375,
    -3.775390625,
    -3.806640625,
    -3.49609375,
    -2.75390625,
    -2.458984375,
    -2.8125,
    -1.986328125,
    -1.4921875,
    -2.056640625,
    -2.265625,
    -2.26171875,
    -1.3828125,
    -2.037109375,
    -3.42578125,
    -2.08984375,
    -1.5751953125,
    -2.529296875,
    -2.220703125,
    -2.123046875,
    -1.765625,
    -1.5869140625,
    -1.6416015625,
    -2.232421875,
    -1.830078125,
    -2.11328125,
    -2.408203125,
    -1.4560546875,
    -2.740234375,
    -2.52734375,
    -0.440185546875,
    -2.38671875,
    -1.1044921875,
    -1.419921875,
    -1.2978515625,
    -1.6416015625,
    -2.861328125,
    -1.6435546875,
    -1.9169921875,
    -2.509765625,
    -2.05859375,
    -2.181640625,
    -1.7373046875,
    -1.533203125,
    -1.478515625,
    -1.3759765625,
    -0.189453125,
    0.12359619140625,
    -0.63427734375,
    0.70849609375,
    1.11328125,
    0.0156707763671875,
    -0.63671875,
    -0.462646484375,
    -0.1090087890625,
    -0.54638671875,
    0.3935546875,
    1.4794921875,
    1.296875,
    -0.67431640625,
    -0.032684326171875,
    -0.97802734375,
    0.283935546875,
    0.57861328125,
    -0.96728515625,
    -0.10736083984375,
    -0.08721923828125,
    0.40966796875,
    0.57666015625,
    -0.1781005859375,
    -0.84375,
    1.2841796875,
    -0.42919921875,
    -0.60107421875,
    -0.332275390625,
    0.465087890625,
    -0.44677734375,
    -2.517578125
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0029787595849484205,
    0.9831173419952393,
    0.006382390856742859,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.007521449122577906
   ],
   "p": [
    0.0030013341456651688,
    0.9905679225921631,
    0.006430760025978088
   ],
   "p_oracle": [
    0.003028564853593707,
    0.9903112053871155,
    0.006660241633653641
   ],
   "abstain": 0.007521449122577906,
   "abstain_oracle": 0.009437600150704384,
   "abs_delta_abstain": 0.0019161510281264782,
   "oracle_margin": 0.9836509823799133,
   "max_abs_delta_p": 0.0002567172050476074,
   "mean_abs_delta_p": 0.00017114317355056605,
   "max_abs_delta_raw_logits": 0.1659531593322754,
   "mean_abs_delta_raw_logits": 0.019905104185454547,
   "logits_min": -8.109375,
   "logits_max": 2.640625,
   "raw_logits_min": -8.109375,
   "raw_logits_max": 2.640625,
   "raw_logits_absmax": 8.109375,
   "oracle_logits_min": -8.139936447143555,
   "oracle_logits_max": 2.5640313625335693,
   "oracle_raw_logits_absmax": 8.139936447143555,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2637140420265496,
   "seconds": 0.2637140420265496,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r09-valve",
   "request_id": "r09",
   "question_id": "valve",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 51,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    3.380859375,
    -2.318359375,
    -2.29296875,
    -2.716796875,
    -2.392578125,
    -2.630859375,
    -3.029296875,
    -4.08984375,
    -3.001953125,
    -2.490234375,
    -2.80859375,
    -3.51953125,
    -4.64453125,
    -2.572265625,
    -4.06640625,
    -4.3828125,
    -4.36328125,
    -4.5859375,
    -2.734375,
    -3.51953125,
    -3.396484375,
    -1.435546875,
    -5.02734375,
    -6.28125,
    -3.826171875,
    -4.484375,
    -2.947265625,
    -5.33984375,
    -4.15234375,
    -6.65234375,
    -4.26953125,
    -5.40625,
    -6.19921875,
    -4.50390625,
    -3.11328125,
    -5.01953125,
    -5.1640625,
    -6.21875,
    -4.31640625,
    -3.685546875,
    -6.671875,
    -4.46875,
    -5.26953125,
    -6.1953125,
    -4.28125,
    -4.85546875,
    -4.19921875,
    -4.93359375,
    -5.1953125,
    -4.12109375,
    -5.546875,
    -4.15625,
    -3.318359375,
    -5.609375,
    -3.1640625,
    -3.37890625,
    -4.15625,
    -4.9921875,
    -4.921875,
    -6.71875,
    -7.54296875,
    -7.17578125,
    -8.140625,
    -6.015625,
    -3.9296875,
    -7.375,
    -7.78515625,
    -2.359375,
    -7.61328125,
    -4.22265625,
    -8.390625,
    -5.421875,
    -3.46875,
    -5.64453125,
    -3.8203125,
    -6.58984375,
    -4.4296875,
    -7.3125,
    -3.978515625,
    -6.1171875,
    -4.80078125,
    -4.14453125,
    -3.19921875,
    -4.60546875,
    -3.646484375,
    -4.44140625,
    -2.66015625,
    -4.1640625,
    -3.244140625,
    -4.92578125,
    -3.890625,
    -4.4765625,
    -4.31640625,
    -3.712890625,
    -4.5859375,
    -4.75,
    -5.0,
    -3.5625,
    -3.283203125,
    -4.28125,
    -5.83203125,
    -5.08203125,
    -3.35546875,
    -4.2578125,
    -4.65625,
    -5.921875,
    -4.15234375,
    -5.34375,
    -4.57421875,
    -6.1640625,
    -4.7578125,
    -4.9375,
    -5.34765625,
    -6.34375,
    -5.17578125,
    -4.09375,
    -4.73046875,
    -6.03515625,
    -4.01171875,
    -4.046875,
    -3.38671875,
    -4.45703125,
    -3.267578125,
    -3.9296875,
    -2.439453125,
    -5.5390625,
    -5.91015625,
    -2.927734375,
    -2.970703125,
    -5.90625,
    -6.30859375,
    -2.890625,
    -5.9296875,
    -6.578125,
    -5.11328125,
    -4.0390625,
    -3.8828125,
    -5.00390625,
    -6.0,
    -2.994140625,
    -3.54296875,
    -4.41796875,
    -3.4765625,
    -2.396484375,
    -5.1875,
    -5.25390625,
    -6.06640625,
    -5.75,
    -5.31640625,
    -4.90234375,
    -4.59765625,
    -2.9453125,
    -1.275390625,
    -3.85546875,
    -3.107421875,
    -1.611328125,
    -1.9580078125,
    -2.01953125,
    -3.009765625,
    -0.35791015625,
    -1.8408203125,
    -2.22265625,
    -1.783203125,
    -0.9072265625,
    -2.994140625,
    -1.9091796875,
    -2.75390625,
    -2.619140625,
    -3.873046875,
    -1.9443359375,
    -2.462890625,
    -2.6328125,
    -2.359375,
    -3.146484375,
    -0.896484375,
    -2.166015625,
    -3.845703125,
    -3.423828125,
    -2.966796875,
    -2.447265625,
    -3.29296875,
    -2.650390625,
    -3.1171875,
    -2.03125,
    -2.59765625,
    -1.8603515625,
    -1.451171875,
    -1.521484375,
    -2.11328125,
    -2.85546875,
    -0.716796875,
    -1.7314453125,
    -2.623046875,
    -1.619140625,
    -1.9345703125,
    -2.044921875,
    -2.275390625,
    -2.11328125,
    -2.291015625,
    -1.439453125,
    -1.359375,
    -2.111328125,
    -1.8408203125,
    -2.078125,
    -2.515625,
    -1.123046875,
    -2.234375,
    -2.623046875,
    0.27587890625,
    -2.10546875,
    -1.4599609375,
    -1.1005859375,
    -0.1910400390625,
    -0.73388671875,
    -2.6171875,
    -1.021484375,
    -1.705078125,
    -1.974609375,
    -1.2587890625,
    -1.787109375,
    -1.0029296875,
    -1.7177734375,
    -0.5732421875,
    -1.1728515625,
    -0.388427734375,
    -0.27587890625,
    -0.420166015625,
    0.87158203125,
    0.89501953125,
    -0.1673583984375,
    -0.414794921875,
    -0.69482421875,
    0.6357421875,
    -0.681640625,
    0.396728515625,
    0.3740234375,
    1.1669921875,
    -0.347412109375,
    0.231201171875,
    -0.7548828125,
    0.27783203125,
    0.80029296875,
    -1.0361328125,
    -0.451904296875,
    -0.226806640625,
    1.07421875,
    0.48974609375,
    -0.8076171875,
    -0.677734375,
    2.306640625,
    0.01287841796875,
    -0.017547607421875,
    -0.328369140625,
    0.31396484375,
    -0.44189453125,
    -4.60546875
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9961764812469482,
    0.0034660438541322947,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003574633738026023
   ],
   "p": [
    0.996532678604126,
    0.003467283211648464
   ],
   "p_oracle": [
    0.9965094327926636,
    0.0034905539359897375
   ],
   "abstain": 0.0003574633738026023,
   "abstain_oracle": 0.0003569536202121526,
   "abs_delta_abstain": 5.097535904496908e-07,
   "oracle_margin": 0.9930188655853271,
   "max_abs_delta_p": 2.3270724341273308e-05,
   "mean_abs_delta_p": 2.3258267901837826e-05,
   "max_abs_delta_raw_logits": 0.029154300689697266,
   "mean_abs_delta_raw_logits": 0.006546195421833545,
   "logits_min": -8.390625,
   "logits_max": 3.380859375,
   "raw_logits_min": -8.390625,
   "raw_logits_max": 3.380859375,
   "raw_logits_absmax": 8.390625,
   "oracle_logits_min": -8.38168716430664,
   "oracle_logits_max": 3.3759267330169678,
   "oracle_raw_logits_absmax": 8.38168716430664,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.29609483294188976,
   "seconds": 0.29609483294188976,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r09-fill",
   "request_id": "r09",
   "question_id": "fill",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 72,
   "nopts": 5,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    -1.87890625,
    -0.24462890625,
    4.1328125,
    -2.044921875,
    -2.97265625,
    -2.525390625,
    -2.52734375,
    -3.671875,
    -2.501953125,
    -2.205078125,
    -3.138671875,
    -3.783203125,
    -4.84375,
    -3.16015625,
    -3.626953125,
    -4.5,
    -3.814453125,
    -4.40234375,
    -2.451171875,
    -3.12890625,
    -3.79296875,
    -1.2490234375,
    -5.01953125,
    -5.84765625,
    -2.93359375,
    -4.41796875,
    -2.80859375,
    -4.02734375,
    -3.8125,
    -6.53125,
    -4.27734375,
    -6.12109375,
    -4.72265625,
    -4.26953125,
    -2.765625,
    -4.95703125,
    -5.20703125,
    -6.6953125,
    -4.6796875,
    -3.359375,
    -6.25390625,
    -4.1484375,
    -4.55078125,
    -6.05078125,
    -4.71875,
    -5.03125,
    -4.50390625,
    -4.2109375,
    -5.22265625,
    -5.05078125,
    -5.296875,
    -5.00390625,
    -2.919921875,
    -5.0078125,
    -3.080078125,
    -3.68359375,
    -4.01171875,
    -5.125,
    -5.0390625,
    -6.58203125,
    -7.45703125,
    -6.72265625,
    -7.5078125,
    -6.10546875,
    -3.32421875,
    -6.94921875,
    -7.4921875,
    -2.82421875,
    -7.84375,
    -4.75390625,
    -7.734375,
    -5.35546875,
    -2.923828125,
    -5.7890625,
    -2.8125,
    -6.2109375,
    -3.802734375,
    -6.640625,
    -4.44921875,
    -5.703125,
    -4.578125,
    -3.62109375,
    -2.525390625,
    -4.37890625,
    -3.412109375,
    -4.3828125,
    -2.642578125,
    -4.12890625,
    -2.890625,
    -4.671875,
    -3.619140625,
    -3.90234375,
    -4.37109375,
    -3.517578125,
    -4.9140625,
    -4.75390625,
    -5.1171875,
    -3.5078125,
    -3.41796875,
    -4.36328125,
    -5.62109375,
    -4.37890625,
    -3.828125,
    -3.640625,
    -4.2109375,
    -5.74609375,
    -3.744140625,
    -5.33984375,
    -4.46875,
    -5.6640625,
    -4.15625,
    -4.296875,
    -5.48828125,
    -6.1171875,
    -4.625,
    -3.859375,
    -4.55078125,
    -5.9375,
    -3.61328125,
    -3.3984375,
    -3.650390625,
    -4.359375,
    -3.234375,
    -3.02734375,
    -2.5390625,
    -5.3984375,
    -5.546875,
    -3.28515625,
    -2.49609375,
    -5.375,
    -5.3828125,
    -2.578125,
    -5.4921875,
    -5.9296875,
    -5.1328125,
    -4.6875,
    -3.623046875,
    -4.64453125,
    -5.65625,
    -3.55859375,
    -3.232421875,
    -4.015625,
    -3.548828125,
    -2.806640625,
    -4.5390625,
    -4.69140625,
    -5.5390625,
    -4.9765625,
    -4.83984375,
    -4.3984375,
    -4.7109375,
    -2.9765625,
    -1.021484375,
    -3.72265625,
    -2.373046875,
    -1.1162109375,
    -1.9248046875,
    -2.095703125,
    -2.98046875,
    0.005931854248046875,
    -1.7451171875,
    -1.8701171875,
    -1.5400390625,
    -1.5078125,
    -3.353515625,
    -2.212890625,
    -3.142578125,
    -1.876953125,
    -3.498046875,
    -2.021484375,
    -2.12109375,
    -2.35546875,
    -2.2265625,
    -2.685546875,
    -0.68603515625,
    -1.927734375,
    -2.71484375,
    -3.150390625,
    -2.66015625,
    -2.85546875,
    -2.6640625,
    -2.88671875,
    -2.85546875,
    -1.9873046875,
    -2.57421875,
    -1.4736328125,
    -2.03515625,
    -1.3857421875,
    -1.63671875,
    -2.833984375,
    -0.75048828125,
    -1.72265625,
    -2.72265625,
    -1.560546875,
    -1.876953125,
    -2.197265625,
    -1.67578125,
    -1.80078125,
    -1.6103515625,
    -1.30859375,
    -1.1416015625,
    -1.8984375,
    -2.056640625,
    -1.8486328125,
    -2.755859375,
    -0.59814453125,
    -2.625,
    -2.40234375,
    0.0180511474609375,
    -2.171875,
    -0.87353515625,
    -1.0771484375,
    -0.25927734375,
    -1.623046875,
    -2.96484375,
    -0.98583984375,
    -1.5546875,
    -2.2265625,
    -1.3427734375,
    -1.6708984375,
    -0.615234375,
    -2.232421875,
    -1.166015625,
    -1.6767578125,
    -0.01324462890625,
    0.1590576171875,
    -0.189208984375,
    1.3193359375,
    0.8896484375,
    -0.358642578125,
    -0.387451171875,
    -0.72705078125,
    0.234375,
    -0.87939453125,
    0.81884765625,
    0.77978515625,
    1.3388671875,
    -0.4501953125,
    0.1134033203125,
    -1.056640625,
    0.64501953125,
    0.46923828125,
    -0.685546875,
    -0.039337158203125,
    -0.11358642578125,
    1.205078125,
    0.61474609375,
    -0.796875,
    -0.89306640625,
    1.9228515625,
    -0.314453125,
    -0.6845703125,
    0.24658203125,
    0.57373046875,
    0.04522705078125,
    -5.80859375
   ],
   "raw256_argmax": 2,
   "option_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "p_full": [
    0.0030736264307051897,
    0.014725005254149437,
    0.978431761264801,
    0.0026213997043669224,
    0.0010771749075502157,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    7.105436088750139e-05
   ],
   "p": [
    0.003073844825848937,
    0.01472605112940073,
    0.9785012602806091,
    0.0026215859688818455,
    0.001077251392416656
   ],
   "p_oracle": [
    0.0030626419465988874,
    0.014278288930654526,
    0.9789782762527466,
    0.002598386723548174,
    0.0010823768097907305
   ],
   "abstain": 7.105436088750139e-05,
   "abstain_oracle": 7.085428660502657e-05,
   "abs_delta_abstain": 2.0007428247481585e-07,
   "oracle_margin": 0.9646999835968018,
   "max_abs_delta_p": 0.00047701597213745117,
   "mean_abs_delta_p": 0.00019286114256829024,
   "max_abs_delta_raw_logits": 0.029170513153076172,
   "mean_abs_delta_raw_logits": 0.005492546304594725,
   "logits_min": -7.84375,
   "logits_max": 4.1328125,
   "raw_logits_min": -7.84375,
   "raw_logits_max": 4.1328125,
   "raw_logits_absmax": 7.84375,
   "oracle_logits_min": -7.844677448272705,
   "oracle_logits_max": 4.1366448402404785,
   "oracle_raw_logits_absmax": 7.844677448272705,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4111389999743551,
   "seconds": 0.4111389999743551,
   "full_vocab_argmax_id": 2,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r10-surface",
   "request_id": "r10",
   "question_id": "surface",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 45,
   "nopts": 4,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.671875,
    4.34765625,
    -2.44140625,
    -3.220703125,
    -2.146484375,
    -2.80078125,
    -2.861328125,
    -3.72265625,
    -3.68359375,
    -2.748046875,
    -3.513671875,
    -4.10546875,
    -5.6796875,
    -3.81640625,
    -4.71484375,
    -4.9140625,
    -3.6953125,
    -4.07421875,
    -3.107421875,
    -3.205078125,
    -3.3671875,
    -1.9013671875,
    -5.23046875,
    -7.2890625,
    -4.05078125,
    -5.5859375,
    -2.578125,
    -5.71484375,
    -4.71484375,
    -7.2109375,
    -4.7578125,
    -5.15625,
    -6.49609375,
    -5.25390625,
    -3.158203125,
    -5.5234375,
    -5.3828125,
    -5.9765625,
    -4.20703125,
    -3.125,
    -7.3828125,
    -4.4453125,
    -4.78515625,
    -6.25,
    -4.91015625,
    -5.328125,
    -4.64453125,
    -4.57421875,
    -5.2890625,
    -4.921875,
    -5.97265625,
    -4.92578125,
    -2.978515625,
    -5.1640625,
    -2.919921875,
    -3.390625,
    -3.68359375,
    -4.62109375,
    -4.91015625,
    -5.890625,
    -7.84765625,
    -6.66015625,
    -7.80078125,
    -6.890625,
    -3.7890625,
    -7.609375,
    -7.796875,
    -2.37890625,
    -8.0703125,
    -4.2578125,
    -8.1484375,
    -5.62109375,
    -2.79296875,
    -6.4140625,
    -3.822265625,
    -6.5625,
    -4.3828125,
    -7.04296875,
    -4.41796875,
    -5.89453125,
    -4.40625,
    -4.3671875,
    -2.837890625,
    -4.375,
    -3.810546875,
    -3.98828125,
    -2.62109375,
    -3.970703125,
    -2.923828125,
    -5.203125,
    -3.71484375,
    -4.34375,
    -4.50390625,
    -4.36328125,
    -4.76953125,
    -4.98046875,
    -5.4609375,
    -4.00390625,
    -4.36328125,
    -4.375,
    -6.48828125,
    -4.671875,
    -3.91796875,
    -4.41796875,
    -4.69140625,
    -6.49609375,
    -4.65234375,
    -5.9765625,
    -4.48046875,
    -5.78125,
    -4.42578125,
    -4.87890625,
    -5.57421875,
    -7.01171875,
    -4.97265625,
    -4.3828125,
    -4.86328125,
    -6.3828125,
    -4.4296875,
    -3.880859375,
    -3.60546875,
    -4.0234375,
    -3.365234375,
    -3.9375,
    -2.560546875,
    -6.15234375,
    -6.09765625,
    -3.486328125,
    -3.341796875,
    -5.8203125,
    -6.0,
    -4.0546875,
    -5.77734375,
    -6.06640625,
    -5.21875,
    -5.0625,
    -4.29296875,
    -5.39453125,
    -6.03125,
    -3.318359375,
    -3.830078125,
    -4.2265625,
    -4.09765625,
    -3.310546875,
    -5.1171875,
    -5.6640625,
    -6.39453125,
    -5.33203125,
    -4.78125,
    -4.71875,
    -4.93359375,
    -3.3984375,
    -1.29296875,
    -3.65234375,
    -2.73046875,
    -1.7314453125,
    -2.248046875,
    -2.130859375,
    -3.578125,
    -0.477294921875,
    -2.150390625,
    -2.095703125,
    -1.798828125,
    -1.3955078125,
    -2.59765625,
    -2.484375,
    -3.076171875,
    -1.884765625,
    -3.544921875,
    -2.052734375,
    -2.44140625,
    -2.63671875,
    -2.62890625,
    -3.078125,
    -0.9794921875,
    -2.04296875,
    -3.392578125,
    -3.52734375,
    -3.33203125,
    -3.810546875,
    -3.494140625,
    -3.541015625,
    -2.841796875,
    -2.173828125,
    -2.673828125,
    -1.7568359375,
    -1.3173828125,
    -2.001953125,
    -2.0078125,
    -2.34375,
    -1.1123046875,
    -1.8779296875,
    -3.181640625,
    -1.9853515625,
    -1.658203125,
    -2.109375,
    -2.158203125,
    -2.181640625,
    -1.69140625,
    -1.537109375,
    -1.271484375,
    -2.078125,
    -1.9375,
    -2.01953125,
    -2.69921875,
    -1.1728515625,
    -3.046875,
    -2.662109375,
    -0.470703125,
    -2.599609375,
    -1.1455078125,
    -1.306640625,
    -0.98974609375,
    -1.8662109375,
    -2.462890625,
    -1.46484375,
    -1.85546875,
    -2.5,
    -1.990234375,
    -1.9599609375,
    -1.23828125,
    -1.830078125,
    -1.4150390625,
    -1.5771484375,
    -0.330322265625,
    0.1895751953125,
    -0.457763671875,
    0.974609375,
    1.2099609375,
    -0.0001653432846069336,
    -1.0556640625,
    -0.5888671875,
    0.2467041015625,
    -0.75390625,
    0.62109375,
    1.177734375,
    1.3974609375,
    -0.75830078125,
    -0.0028171539306640625,
    -0.89453125,
    0.3203125,
    0.74267578125,
    -0.89794921875,
    -0.2353515625,
    0.156005859375,
    0.734375,
    0.35498046875,
    -0.280029296875,
    -1.0078125,
    1.4111328125,
    0.0963134765625,
    -0.8525390625,
    -0.1484375,
    0.42333984375,
    -0.50048828125,
    -3.935546875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.001312915701419115,
    0.9958751797676086,
    0.0016322750598192215,
    0.0007817428559064865,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003979040775448084
   ],
   "p": [
    0.0013134382897987962,
    0.9962716102600098,
    0.0016329248901456594,
    0.0007820540340617299
   ],
   "p_oracle": [
    0.0012890654616057873,
    0.9963335990905762,
    0.0016073038568720222,
    0.0007700907881371677
   ],
   "abstain": 0.0003979040775448084,
   "abstain_oracle": 0.0003932055551558733,
   "abs_delta_abstain": 4.698522388935089e-06,
   "oracle_margin": 0.994726300239563,
   "max_abs_delta_p": 6.198883056640625e-05,
   "mean_abs_delta_p": 3.0986484489403665e-05,
   "max_abs_delta_raw_logits": 0.029924869537353516,
   "mean_abs_delta_raw_logits": 0.004986205603927374,
   "logits_min": -8.1484375,
   "logits_max": 4.34765625,
   "raw_logits_min": -8.1484375,
   "raw_logits_max": 4.34765625,
   "raw_logits_absmax": 8.1484375,
   "oracle_logits_min": -8.147294044494629,
   "oracle_logits_max": 4.35848331451416,
   "oracle_raw_logits_absmax": 8.147294044494629,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2566191670484841,
   "seconds": 0.2566191670484841,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r10-white",
   "request_id": "r10",
   "question_id": "white",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 38,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -3.025390625,
    0.81982421875,
    -1.037109375,
    -1.384765625,
    -1.09375,
    -2.46484375,
    -1.826171875,
    -2.626953125,
    -2.994140625,
    -1.5419921875,
    -2.2578125,
    -3.439453125,
    -4.19921875,
    -2.416015625,
    -3.001953125,
    -4.2890625,
    -3.96484375,
    -4.65234375,
    -2.59375,
    -2.412109375,
    -3.302734375,
    -1.591796875,
    -4.84375,
    -6.14453125,
    -3.560546875,
    -4.44140625,
    -2.421875,
    -4.76171875,
    -3.673828125,
    -5.34765625,
    -4.18359375,
    -3.73046875,
    -5.58203125,
    -4.14453125,
    -2.53515625,
    -4.546875,
    -4.97265625,
    -5.60546875,
    -4.09765625,
    -2.70703125,
    -6.59375,
    -3.9296875,
    -4.4765625,
    -5.76953125,
    -3.52734375,
    -4.38671875,
    -4.1328125,
    -4.34765625,
    -4.609375,
    -4.16015625,
    -5.0703125,
    -4.19140625,
    -2.546875,
    -4.82421875,
    -2.390625,
    -3.41796875,
    -3.34375,
    -4.4140625,
    -4.10546875,
    -5.16796875,
    -6.8125,
    -6.046875,
    -7.234375,
    -5.8984375,
    -3.109375,
    -7.0546875,
    -6.9921875,
    -2.18359375,
    -7.3046875,
    -4.1328125,
    -7.24609375,
    -4.96875,
    -2.2265625,
    -5.37109375,
    -3.580078125,
    -5.5625,
    -4.09765625,
    -6.11328125,
    -3.732421875,
    -5.359375,
    -4.15234375,
    -3.634765625,
    -2.462890625,
    -3.943359375,
    -3.03125,
    -3.865234375,
    -2.384765625,
    -3.259765625,
    -2.755859375,
    -4.10546875,
    -3.16015625,
    -3.71484375,
    -3.3046875,
    -3.87109375,
    -4.35546875,
    -4.62890625,
    -4.515625,
    -3.365234375,
    -3.154296875,
    -4.0234375,
    -5.1640625,
    -4.4921875,
    -3.1171875,
    -3.376953125,
    -4.125,
    -5.35546875,
    -3.201171875,
    -4.8984375,
    -3.83203125,
    -4.953125,
    -4.06640625,
    -4.2265625,
    -4.3515625,
    -5.765625,
    -4.19140625,
    -3.46484375,
    -4.4765625,
    -5.22265625,
    -3.939453125,
    -2.9453125,
    -3.0546875,
    -3.978515625,
    -2.96484375,
    -3.373046875,
    -2.2890625,
    -5.18359375,
    -4.87109375,
    -2.720703125,
    -2.818359375,
    -4.8046875,
    -5.2109375,
    -2.77734375,
    -5.36328125,
    -5.65234375,
    -4.8828125,
    -4.12890625,
    -3.41796875,
    -4.328125,
    -5.42578125,
    -2.259765625,
    -3.14453125,
    -3.927734375,
    -3.3203125,
    -3.201171875,
    -4.1640625,
    -4.68359375,
    -5.3125,
    -4.7890625,
    -4.84375,
    -4.10546875,
    -4.26953125,
    -2.705078125,
    -0.78076171875,
    -2.845703125,
    -2.009765625,
    -1.0322265625,
    -1.4619140625,
    -1.7880859375,
    -3.017578125,
    -0.466552734375,
    -1.853515625,
    -1.939453125,
    -1.1904296875,
    -1.28515625,
    -2.70703125,
    -2.123046875,
    -2.271484375,
    -1.490234375,
    -3.24609375,
    -1.9482421875,
    -1.501953125,
    -2.255859375,
    -1.75390625,
    -2.671875,
    -0.576171875,
    -1.6630859375,
    -2.88671875,
    -3.05859375,
    -2.673828125,
    -2.765625,
    -2.810546875,
    -2.416015625,
    -2.453125,
    -1.8056640625,
    -2.130859375,
    -1.3603515625,
    -1.2841796875,
    -1.3310546875,
    -1.4443359375,
    -1.7607421875,
    -0.491943359375,
    -1.564453125,
    -2.314453125,
    -1.2119140625,
    -1.3515625,
    -1.8115234375,
    -1.7734375,
    -1.224609375,
    -1.330078125,
    -0.97412109375,
    -1.146484375,
    -1.4853515625,
    -1.3857421875,
    -1.4404296875,
    -1.9521484375,
    -0.533203125,
    -2.40234375,
    -2.21484375,
    -0.0849609375,
    -1.630859375,
    -0.478759765625,
    -0.92236328125,
    -0.346435546875,
    -0.875,
    -2.33203125,
    -0.8525390625,
    -1.4248046875,
    -2.115234375,
    -1.4208984375,
    -1.408203125,
    -1.19140625,
    -1.5830078125,
    -1.3720703125,
    -1.150390625,
    -0.08294677734375,
    0.39111328125,
    -0.325927734375,
    1.0126953125,
    1.03125,
    -0.09283447265625,
    -0.646484375,
    -0.373291015625,
    -0.2298583984375,
    -0.87939453125,
    0.48486328125,
    1.2705078125,
    1.599609375,
    -0.57861328125,
    -0.41455078125,
    -0.7333984375,
    0.68017578125,
    0.64111328125,
    -0.69873046875,
    -0.0110931396484375,
    -0.03692626953125,
    0.422607421875,
    0.343505859375,
    -0.6708984375,
    -0.85009765625,
    1.548828125,
    -0.10687255859375,
    -0.849609375,
    0.0021533966064453125,
    0.55078125,
    -0.57080078125,
    -7.41796875
   ],
   "raw256_argmax": 236,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.021464679390192032,
    0.9782618284225464,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00027344783302396536
   ],
   "p": [
    0.021470550447702408,
    0.9785293936729431
   ],
   "p_oracle": [
    0.021688533946871758,
    0.9783114194869995
   ],
   "abstain": 0.00027344783302396536,
   "abstain_oracle": 0.00026848967536352575,
   "abs_delta_abstain": 4.9581576604396105e-06,
   "oracle_margin": 0.9566228985786438,
   "max_abs_delta_p": 0.00021798349916934967,
   "mean_abs_delta_p": 0.0002179788425564766,
   "max_abs_delta_raw_logits": 0.01981937885284424,
   "mean_abs_delta_raw_logits": 0.005654262233292684,
   "logits_min": -7.41796875,
   "logits_max": 1.599609375,
   "raw_logits_min": -7.41796875,
   "raw_logits_max": 1.599609375,
   "raw_logits_absmax": 7.41796875,
   "oracle_logits_min": -7.433703422546387,
   "oracle_logits_max": 1.6013541221618652,
   "oracle_raw_logits_absmax": 7.433703422546387,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.21874562499579042,
   "seconds": 0.21874562499579042,
   "full_vocab_argmax_id": 236,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r10-damage",
   "request_id": "r10",
   "question_id": "damage",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 67,
   "nopts": 5,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    4.8359375,
    -1.228515625,
    -2.18359375,
    -3.4609375,
    -3.177734375,
    -2.931640625,
    -3.283203125,
    -4.203125,
    -3.2890625,
    -2.677734375,
    -2.810546875,
    -3.62109375,
    -4.83203125,
    -3.021484375,
    -4.30078125,
    -4.52734375,
    -4.2265625,
    -4.5078125,
    -2.974609375,
    -3.609375,
    -3.5859375,
    -1.5068359375,
    -5.109375,
    -6.28125,
    -3.91015625,
    -4.37890625,
    -2.97265625,
    -5.28515625,
    -4.16796875,
    -6.70703125,
    -4.1953125,
    -5.70703125,
    -5.76171875,
    -4.6640625,
    -3.2578125,
    -4.91796875,
    -5.0703125,
    -6.2890625,
    -4.20703125,
    -3.53125,
    -6.5703125,
    -4.34765625,
    -5.328125,
    -6.1484375,
    -4.31640625,
    -5.05859375,
    -4.0546875,
    -4.8125,
    -5.375,
    -4.42578125,
    -5.65234375,
    -4.32421875,
    -3.42578125,
    -5.78515625,
    -3.244140625,
    -3.443359375,
    -4.3671875,
    -4.8984375,
    -4.8984375,
    -6.796875,
    -7.6875,
    -7.20703125,
    -8.3515625,
    -5.80859375,
    -3.88671875,
    -7.36328125,
    -8.09375,
    -2.37109375,
    -7.6484375,
    -4.0703125,
    -8.328125,
    -5.55078125,
    -3.619140625,
    -5.69921875,
    -3.70703125,
    -6.55859375,
    -4.375,
    -7.11328125,
    -4.0390625,
    -6.12109375,
    -4.8515625,
    -4.21484375,
    -3.181640625,
    -4.85546875,
    -3.634765625,
    -4.484375,
    -2.466796875,
    -4.2421875,
    -3.181640625,
    -5.11328125,
    -3.759765625,
    -4.44140625,
    -4.328125,
    -3.82421875,
    -4.5859375,
    -4.4765625,
    -5.0859375,
    -3.744140625,
    -3.30078125,
    -4.34375,
    -5.92578125,
    -4.97265625,
    -3.46875,
    -4.328125,
    -4.578125,
    -6.20703125,
    -4.25390625,
    -5.61328125,
    -4.53125,
    -6.25390625,
    -4.828125,
    -4.8515625,
    -5.3828125,
    -6.47265625,
    -5.19140625,
    -4.125,
    -4.84765625,
    -5.87890625,
    -4.04296875,
    -4.28125,
    -3.5078125,
    -4.5078125,
    -3.16796875,
    -3.76171875,
    -2.541015625,
    -5.53515625,
    -5.91796875,
    -3.16796875,
    -3.095703125,
    -6.0703125,
    -6.3671875,
    -2.96875,
    -5.7890625,
    -6.54296875,
    -5.09765625,
    -4.1796875,
    -3.982421875,
    -5.19921875,
    -6.37890625,
    -3.361328125,
    -3.67578125,
    -4.43359375,
    -3.654296875,
    -2.28125,
    -5.41796875,
    -5.55078125,
    -6.17578125,
    -5.72265625,
    -5.16015625,
    -4.95703125,
    -4.69921875,
    -2.83984375,
    -1.2744140625,
    -3.84375,
    -3.017578125,
    -1.66796875,
    -1.8974609375,
    -1.9716796875,
    -2.94921875,
    -0.54736328125,
    -1.771484375,
    -2.244140625,
    -1.9111328125,
    -1.00390625,
    -3.0390625,
    -1.66015625,
    -2.958984375,
    -2.71875,
    -3.90234375,
    -1.9072265625,
    -2.642578125,
    -2.708984375,
    -2.390625,
    -3.119140625,
    -0.96435546875,
    -2.375,
    -4.13671875,
    -3.466796875,
    -2.974609375,
    -2.580078125,
    -3.380859375,
    -2.982421875,
    -3.1953125,
    -2.28515625,
    -2.517578125,
    -1.9169921875,
    -1.2509765625,
    -1.6396484375,
    -2.34765625,
    -3.037109375,
    -0.81689453125,
    -1.8388671875,
    -2.796875,
    -1.8037109375,
    -1.966796875,
    -1.8046875,
    -2.279296875,
    -2.107421875,
    -2.15234375,
    -1.3125,
    -1.232421875,
    -2.08984375,
    -1.7177734375,
    -2.04296875,
    -2.5390625,
    -1.24609375,
    -2.48046875,
    -2.62109375,
    -0.01361083984375,
    -2.40234375,
    -1.8369140625,
    -1.2978515625,
    -0.0193939208984375,
    -0.87109375,
    -2.765625,
    -1.0908203125,
    -1.7919921875,
    -2.017578125,
    -1.25390625,
    -2.0078125,
    -1.0009765625,
    -1.6318359375,
    -0.6435546875,
    -1.18359375,
    -0.3271484375,
    -0.29296875,
    -0.75390625,
    1.0791015625,
    0.875,
    -0.387451171875,
    -0.324462890625,
    -0.66943359375,
    0.8154296875,
    -0.720703125,
    0.383544921875,
    0.509765625,
    1.2021484375,
    -0.26953125,
    0.465576171875,
    -0.88427734375,
    0.136962890625,
    0.81640625,
    -1.0791015625,
    -0.57080078125,
    -0.11981201171875,
    0.91796875,
    0.480224609375,
    -0.42529296875,
    -0.73388671875,
    2.396484375,
    -0.027984619140625,
    -0.01849365234375,
    -0.236572265625,
    0.2078857421875,
    -0.513671875,
    -4.015625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9948261380195618,
    0.0029710670933127403,
    0.0011892726179212332,
    0.0003495268465485424,
    0.0004585509013850242,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00020537413365673274
   ],
   "p": [
    0.9950305819511414,
    0.0029716775752604008,
    0.001189516973681748,
    0.0003495986748021096,
    0.0004586451395880431
   ],
   "p_oracle": [
    0.9949960112571716,
    0.0030125249177217484,
    0.0011956130620092154,
    0.00034420800511725247,
    0.0004516497428994626
   ],
   "abstain": 0.00020537413365673274,
   "abstain_oracle": 0.00020448834402486682,
   "abs_delta_abstain": 8.857896318659186e-07,
   "oracle_margin": 0.9919834733009338,
   "max_abs_delta_p": 4.084734246134758e-05,
   "mean_abs_delta_p": 1.8780038226395844e-05,
   "max_abs_delta_raw_logits": 0.022939682006835938,
   "mean_abs_delta_raw_logits": 0.005487956863362342,
   "logits_min": -8.3515625,
   "logits_max": 4.8359375,
   "raw_logits_min": -8.3515625,
   "raw_logits_max": 4.8359375,
   "raw_logits_absmax": 8.3515625,
   "oracle_logits_min": -8.355578422546387,
   "oracle_logits_max": 4.839208602905273,
   "oracle_raw_logits_absmax": 8.355578422546387,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.37952766695525497,
   "seconds": 0.37952766695525497,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r11-status",
   "request_id": "r11",
   "question_id": "status",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 37,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.8046875,
    3.21875,
    -2.431640625,
    -2.55859375,
    -2.177734375,
    -2.865234375,
    -2.728515625,
    -3.57421875,
    -3.70703125,
    -2.447265625,
    -2.990234375,
    -3.9765625,
    -5.33984375,
    -3.134765625,
    -4.03515625,
    -4.359375,
    -3.697265625,
    -4.31640625,
    -2.919921875,
    -3.05859375,
    -3.41796875,
    -1.994140625,
    -5.27734375,
    -6.87109375,
    -3.90234375,
    -5.359375,
    -2.41015625,
    -5.1953125,
    -4.453125,
    -6.58203125,
    -4.4609375,
    -4.390625,
    -6.1875,
    -4.89453125,
    -3.0625,
    -5.3515625,
    -5.24609375,
    -5.875,
    -4.18359375,
    -2.904296875,
    -7.19140625,
    -4.28125,
    -4.6484375,
    -6.015625,
    -4.25390625,
    -5.04296875,
    -4.5,
    -4.48828125,
    -5.359375,
    -4.65234375,
    -5.67578125,
    -4.68359375,
    -2.73828125,
    -4.86328125,
    -2.654296875,
    -3.333984375,
    -3.533203125,
    -4.40234375,
    -4.70703125,
    -5.43359375,
    -7.54296875,
    -6.3515625,
    -7.62890625,
    -6.5234375,
    -3.61328125,
    -7.4296875,
    -7.67578125,
    -2.3046875,
    -7.7421875,
    -4.07421875,
    -7.78515625,
    -5.28515625,
    -2.474609375,
    -6.1484375,
    -3.724609375,
    -6.1796875,
    -4.2265625,
    -6.67578125,
    -4.23828125,
    -5.65625,
    -4.234375,
    -4.109375,
    -2.66796875,
    -4.21484375,
    -3.51171875,
    -3.90234375,
    -2.583984375,
    -3.80078125,
    -2.9921875,
    -4.84375,
    -3.439453125,
    -4.0625,
    -4.203125,
    -4.19921875,
    -4.58203125,
    -4.65625,
    -5.13671875,
    -3.75,
    -3.9765625,
    -4.14453125,
    -6.046875,
    -4.5078125,
    -3.728515625,
    -4.21484375,
    -4.4765625,
    -5.9765625,
    -4.203125,
    -5.58984375,
    -4.12890625,
    -5.5703125,
    -4.328125,
    -4.59765625,
    -4.94921875,
    -6.4609375,
    -4.74609375,
    -4.015625,
    -4.57421875,
    -6.03125,
    -4.4765625,
    -3.49609375,
    -3.3828125,
    -3.96875,
    -3.275390625,
    -3.783203125,
    -2.37890625,
    -5.828125,
    -5.72265625,
    -3.10546875,
    -3.107421875,
    -5.30859375,
    -5.5546875,
    -3.666015625,
    -5.625,
    -5.953125,
    -4.93359375,
    -4.67578125,
    -3.927734375,
    -4.9453125,
    -5.796875,
    -2.88671875,
    -3.71484375,
    -4.1328125,
    -3.701171875,
    -3.33203125,
    -4.671875,
    -5.19140625,
    -6.08984375,
    -5.140625,
    -4.62109375,
    -4.375,
    -4.734375,
    -3.13671875,
    -1.130859375,
    -3.427734375,
    -2.45703125,
    -1.5966796875,
    -2.01953125,
    -1.92578125,
    -3.412109375,
    -0.4345703125,
    -2.123046875,
    -1.9072265625,
    -1.6650390625,
    -1.2568359375,
    -2.5625,
    -2.302734375,
    -2.865234375,
    -1.76953125,
    -3.4296875,
    -2.021484375,
    -2.296875,
    -2.53515625,
    -2.4375,
    -2.9296875,
    -0.8427734375,
    -1.9345703125,
    -3.19140625,
    -3.28125,
    -3.12890625,
    -3.513671875,
    -3.392578125,
    -3.177734375,
    -2.697265625,
    -2.177734375,
    -2.501953125,
    -1.6376953125,
    -1.326171875,
    -1.75390625,
    -1.919921875,
    -2.0859375,
    -0.9375,
    -1.75,
    -2.849609375,
    -1.701171875,
    -1.390625,
    -2.1484375,
    -2.041015625,
    -1.869140625,
    -1.58984375,
    -1.40234375,
    -1.33984375,
    -1.947265625,
    -1.7197265625,
    -1.89453125,
    -2.263671875,
    -0.96826171875,
    -2.62109375,
    -2.587890625,
    -0.386962890625,
    -2.169921875,
    -0.9130859375,
    -1.2685546875,
    -0.7763671875,
    -1.380859375,
    -2.51953125,
    -1.1875,
    -1.7568359375,
    -2.326171875,
    -1.8173828125,
    -1.9130859375,
    -1.5283203125,
    -1.7099609375,
    -1.462890625,
    -1.330078125,
    -0.21533203125,
    0.107421875,
    -0.48193359375,
    0.83642578125,
    1.076171875,
    -0.03546142578125,
    -0.84326171875,
    -0.480712890625,
    0.034027099609375,
    -0.703125,
    0.55859375,
    1.2607421875,
    1.4638671875,
    -0.7138671875,
    -0.0875244140625,
    -0.90234375,
    0.3818359375,
    0.8291015625,
    -0.8388671875,
    -0.17578125,
    0.04833984375,
    0.4599609375,
    0.315673828125,
    -0.464111328125,
    -0.962890625,
    1.5146484375,
    -0.020111083984375,
    -0.81005859375,
    -0.158935546875,
    0.431640625,
    -0.51806640625,
    -4.2109375
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.003347983118146658,
    0.99100261926651,
    0.004762517753988504,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0008868048898875713
   ],
   "p": [
    0.003350954968482256,
    0.9918822646141052,
    0.004766745027154684
   ],
   "p_oracle": [
    0.0033252013381570578,
    0.9919410347938538,
    0.004733750596642494
   ],
   "abstain": 0.0008868048898875713,
   "abstain_oracle": 0.0008843487594276667,
   "abs_delta_abstain": 2.4561304599046707e-06,
   "oracle_margin": 0.987207293510437,
   "max_abs_delta_p": 5.8770179748535156e-05,
   "mean_abs_delta_p": 3.9172746861974396e-05,
   "max_abs_delta_raw_logits": 0.0246584415435791,
   "mean_abs_delta_raw_logits": 0.0052483148756437,
   "logits_min": -7.78515625,
   "logits_max": 3.21875,
   "raw_logits_min": -7.78515625,
   "raw_logits_max": 3.21875,
   "raw_logits_absmax": 7.78515625,
   "oracle_logits_min": -7.790453910827637,
   "oracle_logits_max": 3.2206735610961914,
   "oracle_raw_logits_absmax": 7.790453910827637,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4124327920144424,
   "seconds": 0.4124327920144424,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r11-restart",
   "request_id": "r11",
   "question_id": "restart",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 42,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    3.22265625,
    -1.8994140625,
    -1.904296875,
    -2.75,
    -2.123046875,
    -2.5859375,
    -2.681640625,
    -3.74609375,
    -2.994140625,
    -2.24609375,
    -2.486328125,
    -3.228515625,
    -4.21484375,
    -2.548828125,
    -3.69140625,
    -4.234375,
    -4.12109375,
    -4.6484375,
    -2.5078125,
    -3.064453125,
    -3.123046875,
    -1.3515625,
    -4.7890625,
    -5.921875,
    -3.416015625,
    -4.1640625,
    -2.708984375,
    -4.87109375,
    -3.857421875,
    -6.3671875,
    -3.9453125,
    -4.7890625,
    -5.90625,
    -4.265625,
    -2.791015625,
    -4.7109375,
    -4.9609375,
    -6.07421875,
    -4.23046875,
    -3.32421875,
    -6.34765625,
    -4.09375,
    -5.15234375,
    -5.921875,
    -4.0625,
    -4.68359375,
    -3.98828125,
    -4.77734375,
    -5.1484375,
    -3.96484375,
    -5.25390625,
    -4.18359375,
    -3.048828125,
    -5.3203125,
    -3.076171875,
    -3.3515625,
    -4.0234375,
    -4.6953125,
    -4.69140625,
    -6.4453125,
    -7.1796875,
    -6.94140625,
    -7.8984375,
    -5.66796875,
    -3.728515625,
    -7.04296875,
    -7.40234375,
    -2.4296875,
    -7.2734375,
    -4.04296875,
    -7.96875,
    -5.0703125,
    -3.216796875,
    -5.34765625,
    -3.533203125,
    -6.14453125,
    -4.03515625,
    -6.91796875,
    -3.69140625,
    -5.73046875,
    -4.59765625,
    -3.83984375,
    -2.876953125,
    -4.4140625,
    -3.43359375,
    -4.33984375,
    -2.365234375,
    -3.96484375,
    -3.044921875,
    -4.5078125,
    -3.509765625,
    -4.2578125,
    -4.12109375,
    -3.3984375,
    -4.29296875,
    -4.4140625,
    -4.80859375,
    -3.30078125,
    -3.001953125,
    -4.1640625,
    -5.35546875,
    -4.76171875,
    -3.236328125,
    -3.904296875,
    -4.53515625,
    -5.65625,
    -3.720703125,
    -5.07421875,
    -4.1953125,
    -5.80859375,
    -4.4921875,
    -4.625,
    -4.953125,
    -6.0,
    -4.78515625,
    -3.9765625,
    -4.6875,
    -5.6171875,
    -3.71484375,
    -3.7734375,
    -3.275390625,
    -4.33984375,
    -3.037109375,
    -3.796875,
    -2.28125,
    -5.21875,
    -5.6484375,
    -2.71875,
    -2.814453125,
    -5.44921875,
    -5.9609375,
    -2.701171875,
    -5.6171875,
    -6.29296875,
    -4.99609375,
    -3.83984375,
    -3.5625,
    -4.70703125,
    -5.8359375,
    -2.783203125,
    -3.279296875,
    -4.171875,
    -3.302734375,
    -2.474609375,
    -4.89453125,
    -5.01953125,
    -5.67578125,
    -5.36328125,
    -4.99609375,
    -4.52734375,
    -4.40234375,
    -2.703125,
    -1.0341796875,
    -3.576171875,
    -2.998046875,
    -1.41015625,
    -1.775390625,
    -1.857421875,
    -2.90625,
    -0.432861328125,
    -1.7578125,
    -2.12890625,
    -1.6982421875,
    -0.87158203125,
    -2.814453125,
    -1.7568359375,
    -2.67578125,
    -2.455078125,
    -3.75390625,
    -1.8759765625,
    -2.228515625,
    -2.48828125,
    -2.1484375,
    -3.0234375,
    -0.560546875,
    -1.9560546875,
    -3.66015625,
    -3.19921875,
    -2.82421875,
    -2.240234375,
    -3.212890625,
    -2.29296875,
    -2.833984375,
    -1.9404296875,
    -2.33984375,
    -1.6650390625,
    -1.3828125,
    -1.455078125,
    -2.15234375,
    -2.5703125,
    -0.4833984375,
    -1.6669921875,
    -2.37109375,
    -1.529296875,
    -1.5859375,
    -1.7236328125,
    -2.126953125,
    -1.8359375,
    -1.98828125,
    -1.0419921875,
    -1.1943359375,
    -1.74609375,
    -1.673828125,
    -1.7646484375,
    -2.265625,
    -0.890625,
    -2.087890625,
    -2.380859375,
    0.1063232421875,
    -1.8408203125,
    -1.2578125,
    -1.0078125,
    0.0479736328125,
    -0.5537109375,
    -2.58203125,
    -0.9404296875,
    -1.591796875,
    -1.84765625,
    -1.07421875,
    -1.7578125,
    -0.83935546875,
    -1.7041015625,
    -0.6884765625,
    -1.08984375,
    -0.2362060546875,
    -0.13916015625,
    -0.5703125,
    1.068359375,
    0.77392578125,
    -0.3046875,
    -0.40625,
    -0.72705078125,
    0.40869140625,
    -0.75048828125,
    0.33837890625,
    0.5234375,
    1.3173828125,
    -0.420654296875,
    0.133544921875,
    -0.76806640625,
    0.434326171875,
    0.82470703125,
    -0.9560546875,
    -0.4736328125,
    -0.13720703125,
    0.83447265625,
    0.57373046875,
    -0.7734375,
    -0.62646484375,
    2.26953125,
    -0.256103515625,
    -0.299072265625,
    -0.390380859375,
    0.262451171875,
    -0.407958984375,
    -4.83203125
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9935342073440552,
    0.006132648792117834,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00033311150036752224
   ],
   "p": [
    0.9938653111457825,
    0.006134692579507828
   ],
   "p_oracle": [
    0.9938108921051025,
    0.006189099978655577
   ],
   "abstain": 0.00033311150036752224,
   "abstain_oracle": 0.00033229435211978853,
   "abs_delta_abstain": 8.171482477337122e-07,
   "oracle_margin": 0.9876217842102051,
   "max_abs_delta_p": 5.441904067993164e-05,
   "mean_abs_delta_p": 5.4413219913840294e-05,
   "max_abs_delta_raw_logits": 0.025194883346557617,
   "mean_abs_delta_raw_logits": 0.005413400416728109,
   "logits_min": -7.96875,
   "logits_max": 3.22265625,
   "raw_logits_min": -7.96875,
   "raw_logits_max": 3.22265625,
   "raw_logits_absmax": 7.96875,
   "oracle_logits_min": -7.964931488037109,
   "oracle_logits_max": 3.2155966758728027,
   "oracle_raw_logits_absmax": 7.964931488037109,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.24432533397339284,
   "seconds": 0.24432533397339284,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r11-urgency",
   "request_id": "r11",
   "question_id": "urgency",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 46,
   "nopts": 3,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    4.09765625,
    -0.8740234375,
    -2.37109375,
    -2.7109375,
    -2.3515625,
    -3.009765625,
    -2.57421875,
    -3.66015625,
    -3.2578125,
    -2.09375,
    -2.697265625,
    -3.42578125,
    -4.37890625,
    -2.732421875,
    -3.794921875,
    -4.03515625,
    -3.853515625,
    -4.5546875,
    -2.869140625,
    -3.2578125,
    -3.0234375,
    -1.5830078125,
    -5.00390625,
    -6.015625,
    -3.341796875,
    -4.19140625,
    -2.490234375,
    -4.68359375,
    -4.00390625,
    -6.39453125,
    -4.046875,
    -4.76171875,
    -5.8671875,
    -4.37890625,
    -2.955078125,
    -4.5859375,
    -5.15625,
    -6.14453125,
    -4.203125,
    -3.20703125,
    -6.5625,
    -4.44921875,
    -5.0390625,
    -5.92578125,
    -3.99609375,
    -4.5625,
    -3.91796875,
    -4.6875,
    -5.359375,
    -4.27734375,
    -5.421875,
    -4.171875,
    -3.119140625,
    -5.46484375,
    -3.09375,
    -3.416015625,
    -3.8828125,
    -4.6796875,
    -4.53515625,
    -6.33984375,
    -7.2578125,
    -6.72265625,
    -7.87109375,
    -5.66796875,
    -3.75,
    -7.11328125,
    -7.45703125,
    -2.30078125,
    -7.23046875,
    -4.0703125,
    -7.83984375,
    -5.1328125,
    -3.0546875,
    -5.40234375,
    -3.685546875,
    -5.9921875,
    -4.0859375,
    -6.78125,
    -3.630859375,
    -5.75,
    -4.50390625,
    -3.900390625,
    -2.84375,
    -4.52734375,
    -3.298828125,
    -4.3515625,
    -2.21484375,
    -4.14453125,
    -2.984375,
    -4.61328125,
    -3.6015625,
    -4.265625,
    -4.09765625,
    -3.560546875,
    -4.296875,
    -4.30078125,
    -4.953125,
    -3.3984375,
    -2.978515625,
    -4.14453125,
    -5.4765625,
    -4.6484375,
    -3.177734375,
    -3.822265625,
    -4.41015625,
    -5.65625,
    -3.708984375,
    -5.27734375,
    -4.05859375,
    -6.0,
    -4.5234375,
    -4.578125,
    -4.921875,
    -6.1328125,
    -4.81640625,
    -3.951171875,
    -4.63671875,
    -5.66796875,
    -3.79296875,
    -3.619140625,
    -3.16796875,
    -4.296875,
    -3.15234375,
    -3.755859375,
    -2.396484375,
    -5.16015625,
    -5.59375,
    -2.662109375,
    -2.966796875,
    -5.46875,
    -5.875,
    -2.705078125,
    -5.546875,
    -6.19140625,
    -4.796875,
    -3.994140625,
    -3.55859375,
    -4.83203125,
    -5.91015625,
    -2.919921875,
    -3.44921875,
    -4.203125,
    -3.208984375,
    -2.521484375,
    -4.96875,
    -5.09375,
    -5.69921875,
    -5.2578125,
    -5.05078125,
    -4.5625,
    -4.30078125,
    -2.8046875,
    -1.083984375,
    -3.515625,
    -2.978515625,
    -1.57421875,
    -1.78515625,
    -1.919921875,
    -2.984375,
    -0.5947265625,
    -1.7021484375,
    -2.48046875,
    -1.7578125,
    -1.0283203125,
    -2.77734375,
    -1.8125,
    -2.671875,
    -2.431640625,
    -3.669921875,
    -1.794921875,
    -2.15625,
    -2.5546875,
    -2.26171875,
    -2.974609375,
    -0.63427734375,
    -2.048828125,
    -3.779296875,
    -3.380859375,
    -2.8359375,
    -2.376953125,
    -3.240234375,
    -2.53125,
    -2.744140625,
    -2.31640625,
    -2.486328125,
    -1.951171875,
    -1.2294921875,
    -1.544921875,
    -2.37890625,
    -2.724609375,
    -0.53271484375,
    -1.7080078125,
    -2.509765625,
    -1.673828125,
    -1.7021484375,
    -1.5478515625,
    -2.0546875,
    -1.8056640625,
    -2.13671875,
    -0.9404296875,
    -1.0830078125,
    -1.8134765625,
    -1.43359375,
    -1.966796875,
    -2.390625,
    -1.080078125,
    -2.35546875,
    -2.533203125,
    0.00035381317138671875,
    -2.115234375,
    -1.3330078125,
    -1.1875,
    -0.01708984375,
    -0.5859375,
    -2.662109375,
    -1.005859375,
    -1.7392578125,
    -1.845703125,
    -1.2783203125,
    -1.9658203125,
    -1.044921875,
    -1.7255859375,
    -0.82861328125,
    -1.1376953125,
    -0.28076171875,
    -0.194580078125,
    -0.7021484375,
    1.37109375,
    0.85302734375,
    -0.312255859375,
    -0.3974609375,
    -0.619140625,
    0.6572265625,
    -0.73291015625,
    0.301025390625,
    0.640625,
    1.3603515625,
    -0.2266845703125,
    0.3408203125,
    -0.9306640625,
    0.302001953125,
    0.59716796875,
    -0.81884765625,
    -0.216796875,
    0.00982666015625,
    0.66357421875,
    0.60107421875,
    -0.4365234375,
    -0.77490234375,
    2.26171875,
    -0.2286376953125,
    -0.28759765625,
    -0.281494140625,
    0.281005859375,
    -0.548828125,
    -4.6640625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9893490076065063,
    0.008423118852078915,
    0.002005360322073102,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0002226141223218292
   ],
   "p": [
    0.9895692467689514,
    0.008424993604421616,
    0.0020058066584169865
   ],
   "p_oracle": [
    0.9895609617233276,
    0.008438709191977978,
    0.0020002825185656548
   ],
   "abstain": 0.0002226141223218292,
   "abstain_oracle": 0.00022156724298838526,
   "abs_delta_abstain": 1.0468793334439397e-06,
   "oracle_margin": 0.9811222553253174,
   "max_abs_delta_p": 1.3715587556362152e-05,
   "mean_abs_delta_p": 9.174924343824387e-06,
   "max_abs_delta_raw_logits": 0.02018904685974121,
   "mean_abs_delta_raw_logits": 0.004825682379305363,
   "logits_min": -7.87109375,
   "logits_max": 4.09765625,
   "raw_logits_min": -7.87109375,
   "raw_logits_max": 4.09765625,
   "raw_logits_absmax": 7.87109375,
   "oracle_logits_min": -7.868396759033203,
   "oracle_logits_max": 4.100948810577393,
   "oracle_raw_logits_absmax": 7.868396759033203,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2655897500226274,
   "seconds": 0.2655897500226274,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r12-direction",
   "request_id": "r12",
   "question_id": "direction",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 44,
   "nopts": 4,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.794921875,
    -2.515625,
    -3.376953125,
    3.314453125,
    -1.908203125,
    -2.26171875,
    -2.83984375,
    -2.912109375,
    -3.3515625,
    -1.9736328125,
    -1.8447265625,
    -2.779296875,
    -3.861328125,
    -1.3935546875,
    -3.77734375,
    -4.82421875,
    -4.50390625,
    -5.0,
    -2.828125,
    -2.287109375,
    -3.82421875,
    -1.4375,
    -5.62109375,
    -5.50390625,
    -4.14453125,
    -4.48046875,
    -2.865234375,
    -5.0234375,
    -3.611328125,
    -5.4453125,
    -4.9453125,
    -4.33203125,
    -6.74609375,
    -3.466796875,
    -2.55859375,
    -5.05078125,
    -5.0,
    -6.0703125,
    -3.828125,
    -3.412109375,
    -6.8203125,
    -3.912109375,
    -5.0078125,
    -6.0078125,
    -3.64453125,
    -4.25390625,
    -4.859375,
    -5.0625,
    -5.421875,
    -3.609375,
    -4.921875,
    -4.12109375,
    -2.67578125,
    -5.62109375,
    -2.341796875,
    -2.365234375,
    -4.4921875,
    -4.88671875,
    -4.6640625,
    -5.9296875,
    -7.2265625,
    -6.67578125,
    -7.76171875,
    -5.80859375,
    -3.564453125,
    -7.01171875,
    -7.36328125,
    -1.833984375,
    -7.4140625,
    -4.47265625,
    -7.86328125,
    -5.24609375,
    -2.3828125,
    -5.26171875,
    -3.71484375,
    -5.98046875,
    -4.515625,
    -6.875,
    -3.619140625,
    -5.3125,
    -4.35546875,
    -3.771484375,
    -2.33203125,
    -4.0703125,
    -3.19140625,
    -4.3046875,
    -2.642578125,
    -3.107421875,
    -3.150390625,
    -4.28515625,
    -3.673828125,
    -4.0078125,
    -3.912109375,
    -3.587890625,
    -4.85546875,
    -4.6171875,
    -4.94140625,
    -3.513671875,
    -3.64453125,
    -3.83984375,
    -5.33203125,
    -5.09765625,
    -3.33984375,
    -3.94921875,
    -4.6015625,
    -5.37109375,
    -3.7109375,
    -4.76171875,
    -4.58203125,
    -5.40234375,
    -4.23828125,
    -4.94140625,
    -4.44921875,
    -5.8203125,
    -4.8984375,
    -4.14453125,
    -4.01171875,
    -5.76171875,
    -3.994140625,
    -2.78125,
    -2.67578125,
    -4.13671875,
    -3.2421875,
    -3.775390625,
    -2.005859375,
    -5.51171875,
    -5.26171875,
    -2.85546875,
    -3.576171875,
    -5.6640625,
    -5.46484375,
    -2.44921875,
    -5.8671875,
    -6.09765625,
    -4.59765625,
    -4.54296875,
    -3.912109375,
    -4.73828125,
    -6.0078125,
    -2.4296875,
    -3.26171875,
    -4.41015625,
    -3.5234375,
    -3.189453125,
    -4.1484375,
    -4.8046875,
    -5.91015625,
    -5.53125,
    -5.67578125,
    -4.77734375,
    -4.296875,
    -2.796875,
    -1.1240234375,
    -3.96484375,
    -2.701171875,
    -1.5390625,
    -1.974609375,
    -1.7646484375,
    -2.955078125,
    -0.99658203125,
    -2.462890625,
    -2.859375,
    -1.6826171875,
    -1.0380859375,
    -3.267578125,
    -2.00390625,
    -2.58203125,
    -2.48046875,
    -4.05078125,
    -2.154296875,
    -1.9560546875,
    -2.345703125,
    -2.244140625,
    -3.107421875,
    -1.0986328125,
    -1.771484375,
    -3.4453125,
    -3.189453125,
    -2.6171875,
    -3.072265625,
    -3.212890625,
    -2.564453125,
    -3.0,
    -1.7490234375,
    -3.029296875,
    -2.11328125,
    -1.7255859375,
    -2.296875,
    -1.6572265625,
    -1.4931640625,
    -1.10546875,
    -1.3193359375,
    -2.595703125,
    -1.66015625,
    -1.5615234375,
    -2.330078125,
    -1.62890625,
    -1.501953125,
    -1.90625,
    -1.1689453125,
    -1.50390625,
    -1.8515625,
    -1.53125,
    -2.068359375,
    -2.1953125,
    -1.3828125,
    -2.037109375,
    -2.5625,
    0.2308349609375,
    -2.2421875,
    -0.82421875,
    -1.265625,
    -0.501953125,
    -1.25390625,
    -2.4375,
    -1.3154296875,
    -1.9130859375,
    -2.173828125,
    -1.5595703125,
    -1.8515625,
    -1.1806640625,
    -1.4326171875,
    -0.94189453125,
    -1.421875,
    0.146484375,
    0.198486328125,
    -0.85498046875,
    0.75390625,
    1.1650390625,
    -0.20068359375,
    -1.236328125,
    -0.69140625,
    -0.261962890625,
    -1.1572265625,
    0.56640625,
    0.716796875,
    1.265625,
    -0.7255859375,
    -0.64453125,
    -0.474853515625,
    0.61376953125,
    0.6943359375,
    -0.78955078125,
    0.287841796875,
    -0.0880126953125,
    0.251220703125,
    0.1590576171875,
    -0.290283203125,
    -0.1268310546875,
    1.59765625,
    -0.01375579833984375,
    -0.482177734375,
    -0.54248046875,
    1.142578125,
    -0.0020694732666015625,
    -5.18359375
   ],
   "raw256_argmax": 3,
   "option_argmax": 3,
   "oracle_argmax": 3,
   "argmax_agrees": true,
   "p_full": [
    0.0030862591229379177,
    0.004018110688775778,
    0.00178088981192559,
    0.9907915592193604,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003231542941648513
   ],
   "p": [
    0.0030872568022459745,
    0.004019409883767366,
    0.0017814654856920242,
    0.9911118745803833
   ],
   "p_oracle": [
    0.003069420112296939,
    0.004049216862767935,
    0.0018126902868971229,
    0.9910686612129211
   ],
   "abstain": 0.0003231542941648513,
   "abstain_oracle": 0.00032492054742760956,
   "abs_delta_abstain": 1.766253262758255e-06,
   "oracle_margin": 0.987019419670105,
   "max_abs_delta_p": 4.32133674621582e-05,
   "mean_abs_delta_p": 3.052045940421522e-05,
   "max_abs_delta_raw_logits": 0.031649112701416016,
   "mean_abs_delta_raw_logits": 0.00520816640346311,
   "logits_min": -7.86328125,
   "logits_max": 3.314453125,
   "raw_logits_min": -7.86328125,
   "raw_logits_max": 3.314453125,
   "raw_logits_absmax": 7.86328125,
   "oracle_logits_min": -7.86134147644043,
   "oracle_logits_max": 3.3078465461730957,
   "oracle_raw_logits_absmax": 7.86134147644043,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2563255000859499,
   "seconds": 0.2563255000859499,
   "full_vocab_argmax_id": 3,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r12-open",
   "request_id": "r12",
   "question_id": "open",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 36,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    2.62109375,
    -2.208984375,
    -1.9609375,
    -2.353515625,
    -1.9423828125,
    -2.287109375,
    -2.6796875,
    -3.373046875,
    -2.869140625,
    -1.923828125,
    -2.498046875,
    -3.123046875,
    -3.833984375,
    -2.396484375,
    -3.638671875,
    -4.2578125,
    -4.125,
    -4.890625,
    -2.501953125,
    -3.005859375,
    -3.232421875,
    -1.439453125,
    -4.83203125,
    -5.88671875,
    -3.193359375,
    -4.09765625,
    -2.77734375,
    -4.71875,
    -3.822265625,
    -6.1171875,
    -3.958984375,
    -4.5703125,
    -5.78125,
    -4.1484375,
    -2.49609375,
    -4.4140625,
    -4.81640625,
    -6.0234375,
    -4.2890625,
    -3.265625,
    -6.2734375,
    -4.0703125,
    -5.0078125,
    -5.84765625,
    -3.751953125,
    -4.33984375,
    -3.8671875,
    -4.76171875,
    -5.0625,
    -3.939453125,
    -5.29296875,
    -4.16796875,
    -3.212890625,
    -5.3125,
    -2.82421875,
    -3.37109375,
    -4.03125,
    -4.6328125,
    -4.54296875,
    -6.21875,
    -7.04296875,
    -6.76953125,
    -7.74609375,
    -5.609375,
    -3.59765625,
    -6.98828125,
    -7.26171875,
    -2.259765625,
    -7.12109375,
    -4.08984375,
    -7.74609375,
    -5.03125,
    -2.974609375,
    -5.265625,
    -3.66015625,
    -5.921875,
    -4.02734375,
    -6.7265625,
    -3.662109375,
    -5.66015625,
    -4.57421875,
    -3.724609375,
    -2.765625,
    -4.37109375,
    -3.275390625,
    -4.19140625,
    -2.220703125,
    -3.84375,
    -3.00390625,
    -4.40234375,
    -3.427734375,
    -4.18359375,
    -4.0859375,
    -3.302734375,
    -4.34375,
    -4.2890625,
    -4.69140625,
    -3.3125,
    -2.91796875,
    -4.109375,
    -5.26171875,
    -4.65625,
    -3.056640625,
    -3.765625,
    -4.3359375,
    -5.48828125,
    -3.498046875,
    -4.8984375,
    -4.08984375,
    -5.94140625,
    -4.40625,
    -4.6171875,
    -4.73828125,
    -5.8359375,
    -4.59765625,
    -3.751953125,
    -4.56640625,
    -5.38671875,
    -3.7578125,
    -3.431640625,
    -3.07421875,
    -4.35546875,
    -3.068359375,
    -3.681640625,
    -2.236328125,
    -5.05078125,
    -5.3828125,
    -2.5234375,
    -2.74609375,
    -5.2734375,
    -5.7578125,
    -2.576171875,
    -5.4765625,
    -6.03125,
    -5.01171875,
    -3.951171875,
    -3.552734375,
    -4.6015625,
    -5.6015625,
    -2.62890625,
    -3.357421875,
    -4.0078125,
    -3.224609375,
    -2.44921875,
    -4.6796875,
    -4.953125,
    -5.421875,
    -5.2109375,
    -5.05078125,
    -4.39453125,
    -4.31640625,
    -2.642578125,
    -0.94580078125,
    -3.36328125,
    -2.908203125,
    -1.3564453125,
    -1.80078125,
    -1.8271484375,
    -2.9609375,
    -0.412353515625,
    -1.7216796875,
    -2.14453125,
    -1.62109375,
    -0.98486328125,
    -2.912109375,
    -1.8173828125,
    -2.6171875,
    -2.322265625,
    -3.6640625,
    -1.91015625,
    -2.046875,
    -2.453125,
    -2.123046875,
    -2.916015625,
    -0.5634765625,
    -1.822265625,
    -3.638671875,
    -3.234375,
    -2.646484375,
    -1.9755859375,
    -3.1171875,
    -2.216796875,
    -2.748046875,
    -1.9228515625,
    -2.255859375,
    -1.716796875,
    -1.412109375,
    -1.380859375,
    -2.060546875,
    -2.275390625,
    -0.254150390625,
    -1.6484375,
    -2.251953125,
    -1.421875,
    -1.5966796875,
    -1.7041015625,
    -2.017578125,
    -1.6279296875,
    -1.9599609375,
    -1.048828125,
    -1.07421875,
    -1.802734375,
    -1.6376953125,
    -1.6689453125,
    -2.185546875,
    -0.76611328125,
    -2.15625,
    -2.306640625,
    0.187255859375,
    -1.8212890625,
    -1.185546875,
    -0.98291015625,
    0.07342529296875,
    -0.468017578125,
    -2.568359375,
    -0.75634765625,
    -1.5703125,
    -1.80078125,
    -1.0869140625,
    -1.56640625,
    -0.87939453125,
    -1.7294921875,
    -0.697265625,
    -1.0458984375,
    -0.299072265625,
    -0.1123046875,
    -0.62939453125,
    1.0732421875,
    0.90576171875,
    -0.048980712890625,
    -0.4111328125,
    -0.67822265625,
    0.363037109375,
    -0.796875,
    0.295654296875,
    0.5458984375,
    1.2900390625,
    -0.2041015625,
    -0.02337646484375,
    -0.8486328125,
    0.4599609375,
    0.78125,
    -0.90576171875,
    -0.21484375,
    -0.2467041015625,
    0.84619140625,
    0.51904296875,
    -0.7763671875,
    -0.64599609375,
    2.177734375,
    -0.2266845703125,
    -0.360595703125,
    -0.277099609375,
    0.358642578125,
    -0.406005859375,
    -5.7109375
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9915677905082703,
    0.00817986112087965,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0002524004376027733
   ],
   "p": [
    0.9918180704116821,
    0.008181925863027573
   ],
   "p_oracle": [
    0.9916988015174866,
    0.008301186375319958
   ],
   "abstain": 0.0002524004376027733,
   "abstain_oracle": 0.000250753277214244,
   "abs_delta_abstain": 1.6471603885293007e-06,
   "oracle_margin": 0.9833976030349731,
   "max_abs_delta_p": 0.00011926889419555664,
   "mean_abs_delta_p": 0.00011926470324397087,
   "max_abs_delta_raw_logits": 0.03310394287109375,
   "mean_abs_delta_raw_logits": 0.00649457226973027,
   "logits_min": -7.74609375,
   "logits_max": 2.62109375,
   "raw_logits_min": -7.74609375,
   "raw_logits_max": 2.62109375,
   "raw_logits_absmax": 7.74609375,
   "oracle_logits_min": -7.743186950683594,
   "oracle_logits_max": 2.6113667488098145,
   "oracle_raw_logits_absmax": 7.743186950683594,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2067403750261292,
   "seconds": 0.2067403750261292,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r12-slope",
   "request_id": "r12",
   "question_id": "slope",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 50,
   "nopts": 3,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    4.1953125,
    -0.9375,
    -2.53125,
    -2.939453125,
    -2.501953125,
    -2.828125,
    -2.8515625,
    -4.078125,
    -3.2890625,
    -2.154296875,
    -2.68359375,
    -3.630859375,
    -4.5859375,
    -2.7421875,
    -3.99609375,
    -4.1875,
    -4.140625,
    -4.61328125,
    -2.814453125,
    -3.39453125,
    -3.291015625,
    -1.6328125,
    -5.12109375,
    -6.34375,
    -3.521484375,
    -4.28125,
    -2.560546875,
    -4.91796875,
    -4.078125,
    -6.47265625,
    -4.0703125,
    -4.8984375,
    -5.8125,
    -4.43359375,
    -2.947265625,
    -4.6875,
    -5.1015625,
    -6.0859375,
    -4.203125,
    -3.38671875,
    -6.67578125,
    -4.390625,
    -5.1015625,
    -6.08203125,
    -4.07421875,
    -4.77734375,
    -3.96875,
    -4.75390625,
    -5.42578125,
    -4.25,
    -5.484375,
    -4.23046875,
    -3.1796875,
    -5.4453125,
    -3.064453125,
    -3.443359375,
    -4.06640625,
    -4.86328125,
    -4.62109375,
    -6.46484375,
    -7.34375,
    -6.8046875,
    -8.0859375,
    -5.6875,
    -3.845703125,
    -7.203125,
    -7.6171875,
    -2.296875,
    -7.34375,
    -4.02734375,
    -7.94140625,
    -5.203125,
    -3.2578125,
    -5.484375,
    -3.7421875,
    -6.16015625,
    -4.14453125,
    -6.91796875,
    -3.78125,
    -5.86328125,
    -4.609375,
    -3.96875,
    -2.896484375,
    -4.6484375,
    -3.4765625,
    -4.2890625,
    -2.2578125,
    -4.04296875,
    -3.03515625,
    -4.83203125,
    -3.61328125,
    -4.3984375,
    -4.21875,
    -3.609375,
    -4.390625,
    -4.28515625,
    -4.9765625,
    -3.53125,
    -3.06640625,
    -4.1640625,
    -5.63671875,
    -4.8046875,
    -3.29296875,
    -3.966796875,
    -4.5234375,
    -5.859375,
    -3.927734375,
    -5.3203125,
    -4.23828125,
    -6.10546875,
    -4.55859375,
    -4.671875,
    -5.11328125,
    -6.140625,
    -4.93359375,
    -4.01171875,
    -4.7890625,
    -5.73046875,
    -3.857421875,
    -3.734375,
    -3.30859375,
    -4.3671875,
    -3.197265625,
    -3.693359375,
    -2.4453125,
    -5.33984375,
    -5.69921875,
    -2.775390625,
    -2.890625,
    -5.671875,
    -6.08203125,
    -2.822265625,
    -5.53125,
    -6.265625,
    -4.88671875,
    -3.982421875,
    -3.693359375,
    -4.953125,
    -6.0703125,
    -3.064453125,
    -3.609375,
    -4.21484375,
    -3.298828125,
    -2.318359375,
    -5.0859375,
    -5.20703125,
    -5.84375,
    -5.36328125,
    -4.9765625,
    -4.625,
    -4.421875,
    -2.806640625,
    -1.044921875,
    -3.666015625,
    -2.896484375,
    -1.611328125,
    -1.810546875,
    -1.9853515625,
    -2.9609375,
    -0.5068359375,
    -1.734375,
    -2.41015625,
    -1.7744140625,
    -0.955078125,
    -2.85546875,
    -1.765625,
    -2.7734375,
    -2.56640625,
    -3.66015625,
    -1.8125,
    -2.2890625,
    -2.564453125,
    -2.3828125,
    -3.0234375,
    -0.76123046875,
    -2.14453125,
    -3.8828125,
    -3.380859375,
    -2.8671875,
    -2.369140625,
    -3.224609375,
    -2.736328125,
    -2.84765625,
    -2.2421875,
    -2.4921875,
    -1.939453125,
    -1.177734375,
    -1.453125,
    -2.388671875,
    -2.736328125,
    -0.53466796875,
    -1.7685546875,
    -2.544921875,
    -1.712890625,
    -1.82421875,
    -1.6669921875,
    -2.19140625,
    -1.9228515625,
    -2.17578125,
    -1.05859375,
    -1.1455078125,
    -1.8349609375,
    -1.4501953125,
    -1.9443359375,
    -2.37109375,
    -1.119140625,
    -2.3828125,
    -2.560546875,
    0.01410675048828125,
    -2.123046875,
    -1.4853515625,
    -1.2080078125,
    0.04156494140625,
    -0.68798828125,
    -2.681640625,
    -0.99658203125,
    -1.7587890625,
    -1.9091796875,
    -1.271484375,
    -1.962890625,
    -1.0439453125,
    -1.7802734375,
    -0.72802734375,
    -1.1474609375,
    -0.302001953125,
    -0.1763916015625,
    -0.68896484375,
    1.23046875,
    0.92578125,
    -0.2138671875,
    -0.418212890625,
    -0.67041015625,
    0.62109375,
    -0.71923828125,
    0.319091796875,
    0.53564453125,
    1.302734375,
    -0.22119140625,
    0.3525390625,
    -0.89501953125,
    0.331298828125,
    0.734375,
    -0.81494140625,
    -0.251953125,
    -0.1351318359375,
    0.71728515625,
    0.60595703125,
    -0.440185546875,
    -0.86572265625,
    2.263671875,
    -0.1912841796875,
    -0.2264404296875,
    -0.171142578125,
    0.273193359375,
    -0.438720703125,
    -4.46875
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9909570217132568,
    0.007229252252727747,
    0.0015687800478190184,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00024485893663950264
   ],
   "p": [
    0.991199791431427,
    0.007231023162603378,
    0.001569164334796369
   ],
   "p_oracle": [
    0.9910877346992493,
    0.007320776116102934,
    0.0015914562391117215
   ],
   "abstain": 0.00024485893663950264,
   "abstain_oracle": 0.0002448532613925636,
   "abs_delta_abstain": 5.675246939063072e-09,
   "oracle_margin": 0.9837669730186462,
   "max_abs_delta_p": 0.00011205673217773438,
   "mean_abs_delta_p": 7.470052999754746e-05,
   "max_abs_delta_raw_logits": 0.026476383209228516,
   "mean_abs_delta_raw_logits": 0.005880058481125161,
   "logits_min": -8.0859375,
   "logits_max": 4.1953125,
   "raw_logits_min": -8.0859375,
   "raw_logits_max": 4.1953125,
   "raw_logits_absmax": 8.0859375,
   "oracle_logits_min": -8.079290390014648,
   "oracle_logits_max": 4.1894683837890625,
   "oracle_raw_logits_absmax": 8.079290390014648,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.28552045894321054,
   "seconds": 0.28552045894321054,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r13-door",
   "request_id": "r13",
   "question_id": "door",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 66,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.31640625,
    3.4375,
    -2.955078125,
    -2.65234375,
    -1.90234375,
    -2.919921875,
    -2.91796875,
    -3.83984375,
    -4.0234375,
    -3.029296875,
    -3.263671875,
    -4.25,
    -5.71875,
    -3.658203125,
    -4.5390625,
    -5.01953125,
    -4.05859375,
    -4.359375,
    -3.2109375,
    -3.255859375,
    -3.49609375,
    -1.9970703125,
    -5.1171875,
    -7.296875,
    -4.22265625,
    -5.48828125,
    -2.736328125,
    -5.84375,
    -4.5,
    -7.234375,
    -4.69140625,
    -5.0,
    -6.453125,
    -5.28515625,
    -3.208984375,
    -5.55078125,
    -5.1484375,
    -5.90625,
    -4.25390625,
    -3.2421875,
    -7.34375,
    -4.49609375,
    -4.97265625,
    -6.28515625,
    -4.75390625,
    -5.2890625,
    -4.68359375,
    -4.70703125,
    -5.20703125,
    -4.74609375,
    -6.0078125,
    -4.7578125,
    -2.892578125,
    -5.16796875,
    -2.6796875,
    -3.1953125,
    -3.783203125,
    -4.796875,
    -4.890625,
    -5.92578125,
    -7.80859375,
    -6.70703125,
    -7.8515625,
    -6.79296875,
    -3.83203125,
    -7.6015625,
    -7.8203125,
    -2.337890625,
    -7.9453125,
    -4.171875,
    -8.3828125,
    -5.5859375,
    -2.873046875,
    -6.2578125,
    -3.87890625,
    -6.67578125,
    -4.5546875,
    -7.140625,
    -4.3046875,
    -5.93359375,
    -4.38671875,
    -4.375,
    -2.8671875,
    -4.29296875,
    -3.662109375,
    -4.05078125,
    -2.73828125,
    -3.955078125,
    -3.091796875,
    -5.34375,
    -3.63671875,
    -4.2578125,
    -4.359375,
    -4.24609375,
    -4.796875,
    -5.0234375,
    -5.26953125,
    -4.12890625,
    -4.41015625,
    -4.23828125,
    -6.28515625,
    -4.84765625,
    -3.619140625,
    -4.46875,
    -4.70703125,
    -6.3671875,
    -4.61328125,
    -5.859375,
    -4.35546875,
    -5.8125,
    -4.47265625,
    -5.12109375,
    -5.52734375,
    -6.9609375,
    -5.0,
    -4.34375,
    -4.80859375,
    -6.19921875,
    -4.5859375,
    -3.822265625,
    -3.537109375,
    -4.015625,
    -3.41015625,
    -3.97265625,
    -2.6640625,
    -6.1640625,
    -6.0546875,
    -3.3984375,
    -3.236328125,
    -5.71875,
    -5.90234375,
    -4.01953125,
    -5.8984375,
    -6.16796875,
    -5.2578125,
    -4.890625,
    -4.33984375,
    -5.1484375,
    -6.14453125,
    -3.162109375,
    -3.8671875,
    -4.234375,
    -3.9921875,
    -3.31640625,
    -5.140625,
    -5.6328125,
    -6.3828125,
    -5.41796875,
    -4.9921875,
    -4.6953125,
    -4.9296875,
    -3.37890625,
    -1.1796875,
    -3.6328125,
    -2.76953125,
    -1.7822265625,
    -2.123046875,
    -2.189453125,
    -3.4375,
    -0.5849609375,
    -2.328125,
    -2.126953125,
    -1.7744140625,
    -1.2294921875,
    -2.697265625,
    -2.5859375,
    -2.890625,
    -1.9853515625,
    -3.4765625,
    -2.126953125,
    -2.533203125,
    -2.634765625,
    -2.6171875,
    -3.224609375,
    -1.099609375,
    -2.029296875,
    -3.353515625,
    -3.375,
    -3.19140625,
    -3.615234375,
    -3.498046875,
    -3.4765625,
    -2.845703125,
    -2.248046875,
    -2.640625,
    -1.71875,
    -1.3310546875,
    -1.9775390625,
    -1.9921875,
    -2.236328125,
    -1.154296875,
    -1.90234375,
    -3.017578125,
    -1.787109375,
    -1.55859375,
    -2.142578125,
    -2.1484375,
    -1.9130859375,
    -1.8359375,
    -1.4921875,
    -1.240234375,
    -2.09765625,
    -1.8671875,
    -1.9267578125,
    -2.583984375,
    -1.326171875,
    -2.794921875,
    -2.7109375,
    -0.43505859375,
    -2.4609375,
    -1.025390625,
    -1.3115234375,
    -0.91455078125,
    -1.640625,
    -2.515625,
    -1.3466796875,
    -1.8974609375,
    -2.37109375,
    -1.87890625,
    -1.9970703125,
    -1.396484375,
    -1.83203125,
    -1.435546875,
    -1.4326171875,
    -0.254638671875,
    0.1334228515625,
    -0.336669921875,
    0.9111328125,
    1.052734375,
    -0.0118408203125,
    -0.8740234375,
    -0.6669921875,
    0.163818359375,
    -0.81103515625,
    0.5927734375,
    1.005859375,
    1.5009765625,
    -0.5791015625,
    0.0816650390625,
    -0.7109375,
    0.30126953125,
    0.8525390625,
    -0.87646484375,
    -0.219970703125,
    0.1724853515625,
    0.67724609375,
    0.2095947265625,
    -0.365478515625,
    -0.970703125,
    1.529296875,
    0.01039886474609375,
    -0.6953125,
    -0.1177978515625,
    0.482177734375,
    -0.5107421875,
    -4.53515625
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.004326555412262678,
    0.9927750825881958,
    0.002366511384025216,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0005319062620401382
   ],
   "p": [
    0.004328857641667128,
    0.9933033585548401,
    0.00236777076497674
   ],
   "p_oracle": [
    0.0043136803433299065,
    0.9933029413223267,
    0.002383379964157939
   ],
   "abstain": 0.0005319062620401382,
   "abstain_oracle": 0.0005352262523956597,
   "abs_delta_abstain": 3.3199903555214405e-06,
   "oracle_margin": 0.9889892339706421,
   "max_abs_delta_p": 1.5609199181199074e-05,
   "mean_abs_delta_p": 1.0401243343949318e-05,
   "max_abs_delta_raw_logits": 0.026207923889160156,
   "mean_abs_delta_raw_logits": 0.00646539794979617,
   "logits_min": -8.3828125,
   "logits_max": 3.4375,
   "raw_logits_min": -8.3828125,
   "raw_logits_max": 3.4375,
   "raw_logits_absmax": 8.3828125,
   "oracle_logits_min": -8.369766235351562,
   "oracle_logits_max": 3.435242176055908,
   "oracle_raw_logits_absmax": 8.369766235351562,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3743030000478029,
   "seconds": 0.3743030000478029,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r13-light",
   "request_id": "r13",
   "question_id": "light",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 65,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -3.251953125,
    1.953125,
    -1.7021484375,
    -1.6708984375,
    -1.75,
    -2.861328125,
    -2.55859375,
    -3.18359375,
    -3.427734375,
    -2.427734375,
    -2.478515625,
    -3.650390625,
    -4.65234375,
    -2.30859375,
    -3.392578125,
    -4.53515625,
    -4.05078125,
    -4.8671875,
    -2.998046875,
    -2.8203125,
    -3.4453125,
    -1.5986328125,
    -5.01171875,
    -6.828125,
    -3.919921875,
    -4.97265625,
    -2.55859375,
    -5.5625,
    -3.978515625,
    -6.38671875,
    -4.51171875,
    -4.56640625,
    -6.109375,
    -4.58203125,
    -2.875,
    -5.13671875,
    -5.17578125,
    -5.81640625,
    -4.20703125,
    -3.18359375,
    -6.95703125,
    -4.203125,
    -4.72265625,
    -6.11328125,
    -4.0703125,
    -4.8125,
    -4.26171875,
    -4.51171875,
    -4.95703125,
    -4.4140625,
    -5.5078125,
    -4.44140625,
    -2.654296875,
    -4.8046875,
    -2.455078125,
    -3.091796875,
    -3.615234375,
    -4.71484375,
    -4.53515625,
    -5.41796875,
    -7.2890625,
    -6.40625,
    -7.3984375,
    -6.3125,
    -3.455078125,
    -7.33203125,
    -7.484375,
    -2.279296875,
    -7.61328125,
    -4.328125,
    -7.94921875,
    -5.09765625,
    -2.49609375,
    -5.94921875,
    -3.759765625,
    -6.140625,
    -4.4140625,
    -6.69921875,
    -3.9453125,
    -5.4921875,
    -4.15234375,
    -3.9609375,
    -2.673828125,
    -3.96484375,
    -3.33984375,
    -4.1015625,
    -2.431640625,
    -3.494140625,
    -2.80078125,
    -4.8046875,
    -3.560546875,
    -3.953125,
    -3.6875,
    -4.1484375,
    -4.67578125,
    -4.82421875,
    -4.88671875,
    -3.85546875,
    -3.75390625,
    -4.0703125,
    -5.6953125,
    -4.78125,
    -3.384765625,
    -3.916015625,
    -4.2421875,
    -5.84765625,
    -3.9296875,
    -5.3984375,
    -4.20703125,
    -5.203125,
    -4.31640625,
    -4.671875,
    -4.9765625,
    -6.36328125,
    -4.71875,
    -3.9296875,
    -4.5,
    -5.765625,
    -4.27734375,
    -3.37109375,
    -3.2734375,
    -3.93359375,
    -3.4296875,
    -3.57421875,
    -2.51953125,
    -5.81640625,
    -5.296875,
    -3.193359375,
    -2.900390625,
    -5.3359375,
    -5.7265625,
    -3.29296875,
    -5.484375,
    -5.94921875,
    -5.18359375,
    -4.57421875,
    -3.716796875,
    -4.8515625,
    -5.63671875,
    -2.833984375,
    -3.521484375,
    -4.01953125,
    -3.732421875,
    -3.31640625,
    -4.77734375,
    -5.0625,
    -5.85546875,
    -5.12109375,
    -5.03125,
    -4.37109375,
    -4.55859375,
    -3.15625,
    -0.96826171875,
    -3.306640625,
    -2.50390625,
    -1.4755859375,
    -1.7861328125,
    -2.21875,
    -3.302734375,
    -0.371337890625,
    -2.185546875,
    -2.005859375,
    -1.599609375,
    -1.1904296875,
    -2.6953125,
    -2.45703125,
    -2.705078125,
    -1.6259765625,
    -3.2109375,
    -2.044921875,
    -2.025390625,
    -2.43359375,
    -2.31640625,
    -2.98046875,
    -0.9580078125,
    -1.83203125,
    -2.904296875,
    -3.2421875,
    -2.9296875,
    -3.337890625,
    -3.126953125,
    -3.015625,
    -2.890625,
    -1.9775390625,
    -2.3203125,
    -1.5771484375,
    -1.4296875,
    -1.8134765625,
    -1.6376953125,
    -2.103515625,
    -0.9287109375,
    -1.44140625,
    -2.72265625,
    -1.4423828125,
    -1.5380859375,
    -2.099609375,
    -2.041015625,
    -1.8232421875,
    -1.6572265625,
    -1.2177734375,
    -1.166015625,
    -1.7861328125,
    -1.6884765625,
    -1.47265625,
    -2.408203125,
    -0.9384765625,
    -2.568359375,
    -2.587890625,
    -0.1134033203125,
    -2.16015625,
    -0.7099609375,
    -1.037109375,
    -0.673828125,
    -1.06640625,
    -2.48046875,
    -1.126953125,
    -1.716796875,
    -2.216796875,
    -1.71484375,
    -1.662109375,
    -1.2685546875,
    -1.7509765625,
    -1.3720703125,
    -1.423828125,
    -0.2161865234375,
    0.338134765625,
    -0.062225341796875,
    0.82666015625,
    0.95166015625,
    -0.10888671875,
    -0.962890625,
    -0.70849609375,
    0.048004150390625,
    -0.806640625,
    0.537109375,
    1.1474609375,
    1.4501953125,
    -0.625,
    -0.040985107421875,
    -0.8681640625,
    0.64453125,
    0.7763671875,
    -0.8095703125,
    -0.2381591796875,
    -0.1038818359375,
    0.50341796875,
    0.125,
    -0.48046875,
    -0.9619140625,
    1.583984375,
    0.054290771484375,
    -0.83251953125,
    0.01291656494140625,
    0.56005859375,
    -0.58447265625,
    -6.1171875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.005650067701935768,
    0.9940218329429626,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003281424578744918
   ],
   "p": [
    0.005651922430843115,
    0.9943481087684631
   ],
   "p_oracle": [
    0.005548183340579271,
    0.9944518208503723
   ],
   "abstain": 0.0003281424578744918,
   "abstain_oracle": 0.0003229013236705214,
   "abs_delta_abstain": 5.241134203970432e-06,
   "oracle_margin": 0.9889036417007446,
   "max_abs_delta_p": 0.00010373909026384354,
   "mean_abs_delta_p": 0.00010372558608651161,
   "max_abs_delta_raw_logits": 0.02547430992126465,
   "mean_abs_delta_raw_logits": 0.004979828838258982,
   "logits_min": -7.94921875,
   "logits_max": 1.953125,
   "raw_logits_min": -7.94921875,
   "raw_logits_max": 1.953125,
   "raw_logits_absmax": 7.94921875,
   "oracle_logits_min": -7.945124626159668,
   "oracle_logits_max": 1.9632949829101562,
   "oracle_raw_logits_absmax": 7.945124626159668,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.36571258294861764,
   "seconds": 0.36571258294861764,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r13-temperature",
   "request_id": "r13",
   "question_id": "temperature",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 83,
   "nopts": 3,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    -1.9833984375,
    3.65234375,
    -2.48828125,
    -2.630859375,
    -1.7822265625,
    -2.6171875,
    -2.736328125,
    -3.9921875,
    -3.689453125,
    -2.68359375,
    -3.419921875,
    -4.39453125,
    -5.5546875,
    -3.55859375,
    -4.49609375,
    -5.09375,
    -3.958984375,
    -4.37890625,
    -3.064453125,
    -3.1484375,
    -3.302734375,
    -1.70703125,
    -5.1015625,
    -7.26953125,
    -4.1640625,
    -5.375,
    -2.587890625,
    -5.96875,
    -4.4609375,
    -7.32421875,
    -4.734375,
    -5.23828125,
    -6.53125,
    -5.22265625,
    -3.0,
    -5.515625,
    -5.1796875,
    -6.03125,
    -4.3046875,
    -3.25,
    -7.4453125,
    -4.546875,
    -5.0234375,
    -6.3046875,
    -4.78515625,
    -5.21484375,
    -4.7578125,
    -4.6796875,
    -5.2890625,
    -4.66796875,
    -6.0234375,
    -4.765625,
    -2.744140625,
    -5.23828125,
    -2.68359375,
    -3.306640625,
    -3.798828125,
    -4.984375,
    -4.9609375,
    -6.19921875,
    -7.8046875,
    -6.73046875,
    -7.92578125,
    -6.8515625,
    -3.6796875,
    -7.6171875,
    -7.76171875,
    -2.412109375,
    -7.88671875,
    -4.234375,
    -8.3984375,
    -5.57421875,
    -2.912109375,
    -6.265625,
    -3.9140625,
    -6.6015625,
    -4.40625,
    -7.17578125,
    -4.234375,
    -5.875,
    -4.44921875,
    -4.29296875,
    -2.828125,
    -4.1796875,
    -3.623046875,
    -4.1171875,
    -2.775390625,
    -3.958984375,
    -2.921875,
    -5.30859375,
    -3.65625,
    -4.23828125,
    -4.37890625,
    -4.25390625,
    -4.76953125,
    -4.98046875,
    -5.26171875,
    -4.12890625,
    -4.43359375,
    -4.20703125,
    -6.296875,
    -4.7890625,
    -3.435546875,
    -4.40234375,
    -4.57421875,
    -6.390625,
    -4.62890625,
    -5.9453125,
    -4.3828125,
    -5.73828125,
    -4.4296875,
    -4.9765625,
    -5.6953125,
    -7.1640625,
    -4.9375,
    -4.3984375,
    -4.875,
    -6.24609375,
    -4.421875,
    -3.73046875,
    -3.587890625,
    -4.12890625,
    -3.564453125,
    -3.90625,
    -2.703125,
    -6.16015625,
    -6.0,
    -3.49609375,
    -3.220703125,
    -5.921875,
    -6.05859375,
    -3.861328125,
    -5.84375,
    -6.21484375,
    -5.2890625,
    -4.6953125,
    -4.2421875,
    -5.32421875,
    -6.1015625,
    -3.24609375,
    -3.8515625,
    -4.13671875,
    -4.03125,
    -3.234375,
    -5.21875,
    -5.76171875,
    -6.44140625,
    -5.43359375,
    -4.96484375,
    -4.90625,
    -4.94921875,
    -3.5546875,
    -1.142578125,
    -3.7890625,
    -2.8515625,
    -1.7275390625,
    -2.228515625,
    -2.412109375,
    -3.580078125,
    -0.44091796875,
    -2.287109375,
    -2.3828125,
    -1.78515625,
    -1.3115234375,
    -2.853515625,
    -2.52734375,
    -2.888671875,
    -1.90625,
    -3.43359375,
    -2.0546875,
    -2.375,
    -2.599609375,
    -2.658203125,
    -3.3046875,
    -1.1787109375,
    -2.09765625,
    -3.408203125,
    -3.3984375,
    -3.06640625,
    -3.552734375,
    -3.33984375,
    -3.419921875,
    -2.861328125,
    -2.072265625,
    -2.591796875,
    -1.5908203125,
    -1.2626953125,
    -1.93359375,
    -1.8515625,
    -2.328125,
    -1.0107421875,
    -1.7138671875,
    -2.927734375,
    -1.6640625,
    -1.748046875,
    -2.150390625,
    -2.03515625,
    -1.8857421875,
    -1.72265625,
    -1.4501953125,
    -1.1142578125,
    -1.9775390625,
    -1.822265625,
    -1.9384765625,
    -2.630859375,
    -1.189453125,
    -2.90625,
    -2.66796875,
    -0.3466796875,
    -2.494140625,
    -0.982421875,
    -1.1630859375,
    -0.916015625,
    -1.681640625,
    -2.4453125,
    -1.2734375,
    -1.8720703125,
    -2.44140625,
    -1.939453125,
    -1.9541015625,
    -1.1005859375,
    -1.8251953125,
    -1.357421875,
    -1.62890625,
    -0.333984375,
    0.2125244140625,
    -0.2318115234375,
    0.880859375,
    1.158203125,
    -0.0139923095703125,
    -0.94970703125,
    -0.72705078125,
    0.251708984375,
    -0.75830078125,
    0.568359375,
    1.0478515625,
    1.548828125,
    -0.513671875,
    0.1705322265625,
    -0.81689453125,
    0.342529296875,
    0.80615234375,
    -0.859375,
    -0.25927734375,
    0.16552734375,
    0.75244140625,
    0.214599609375,
    -0.399658203125,
    -1.0498046875,
    1.568359375,
    0.10015869140625,
    -0.7666015625,
    -0.06793212890625,
    0.433837890625,
    -0.5068359375,
    -5.17578125
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.004471060819923878,
    0.992563784122467,
    0.002755566732957959,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00020956317894160748
   ],
   "p": [
    0.004471998196095228,
    0.9927718639373779,
    0.0027561443857848644
   ],
   "p_oracle": [
    0.004428014624863863,
    0.9928196668624878,
    0.002752336673438549
   ],
   "abstain": 0.00020956317894160748,
   "abstain_oracle": 0.00020866226986981928,
   "abs_delta_abstain": 9.009090717881918e-07,
   "oracle_margin": 0.988391637802124,
   "max_abs_delta_p": 4.780292510986328e-05,
   "mean_abs_delta_p": 3.186473622918129e-05,
   "max_abs_delta_raw_logits": 0.029097795486450195,
   "mean_abs_delta_raw_logits": 0.005187240109080449,
   "logits_min": -8.3984375,
   "logits_max": 3.65234375,
   "raw_logits_min": -8.3984375,
   "raw_logits_max": 3.65234375,
   "raw_logits_absmax": 8.3984375,
   "oracle_logits_min": -8.394624710083008,
   "oracle_logits_max": 3.6570920944213867,
   "oracle_raw_logits_absmax": 8.394624710083008,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4634927079314366,
   "seconds": 0.4634927079314366,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r14-color",
   "request_id": "r14",
   "question_id": "color",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 71,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    3.5078125,
    -2.453125,
    -2.462890625,
    -2.65234375,
    -2.40625,
    -2.541015625,
    -2.673828125,
    -3.828125,
    -3.41796875,
    -2.197265625,
    -2.58984375,
    -3.349609375,
    -4.62109375,
    -2.703125,
    -3.8828125,
    -3.90234375,
    -4.05078125,
    -4.61328125,
    -2.8984375,
    -3.248046875,
    -3.208984375,
    -1.8740234375,
    -5.21484375,
    -6.0,
    -3.826171875,
    -4.27734375,
    -2.935546875,
    -4.74609375,
    -4.0703125,
    -6.15234375,
    -3.986328125,
    -4.71484375,
    -6.0,
    -4.2265625,
    -3.150390625,
    -4.6484375,
    -5.13671875,
    -6.18359375,
    -4.296875,
    -3.2890625,
    -6.28125,
    -4.41796875,
    -5.1796875,
    -6.078125,
    -3.859375,
    -4.63671875,
    -4.11328125,
    -4.76171875,
    -5.30078125,
    -4.125,
    -5.16796875,
    -4.08203125,
    -3.1796875,
    -5.55859375,
    -3.138671875,
    -3.5,
    -4.07421875,
    -4.83984375,
    -4.64453125,
    -6.17578125,
    -7.38671875,
    -6.8828125,
    -7.96875,
    -5.6015625,
    -3.9140625,
    -7.18359375,
    -7.55859375,
    -2.51171875,
    -7.23046875,
    -3.96484375,
    -7.94140625,
    -5.30078125,
    -3.123046875,
    -5.4296875,
    -3.6796875,
    -6.140625,
    -4.24609375,
    -6.87109375,
    -3.873046875,
    -6.0546875,
    -4.484375,
    -4.08203125,
    -2.9765625,
    -4.4140625,
    -3.359375,
    -4.484375,
    -2.4140625,
    -4.1015625,
    -3.10546875,
    -4.60546875,
    -3.779296875,
    -4.25390625,
    -4.28515625,
    -3.55859375,
    -4.38671875,
    -4.37890625,
    -4.953125,
    -3.306640625,
    -2.896484375,
    -4.2109375,
    -5.55859375,
    -4.69140625,
    -3.271484375,
    -3.974609375,
    -4.59375,
    -5.66015625,
    -3.71484375,
    -5.09765625,
    -4.33203125,
    -5.9140625,
    -4.43359375,
    -4.765625,
    -5.15625,
    -5.9765625,
    -4.84765625,
    -3.751953125,
    -4.65625,
    -5.66015625,
    -3.833984375,
    -3.66796875,
    -3.142578125,
    -4.31640625,
    -3.080078125,
    -3.68359375,
    -2.404296875,
    -5.2421875,
    -5.765625,
    -2.54296875,
    -2.916015625,
    -5.671875,
    -5.96484375,
    -2.771484375,
    -5.6953125,
    -6.42578125,
    -4.9765625,
    -4.06640625,
    -3.7734375,
    -4.66015625,
    -6.09375,
    -2.88671875,
    -3.517578125,
    -4.23828125,
    -3.26953125,
    -2.484375,
    -4.97265625,
    -5.08984375,
    -5.76171875,
    -5.49609375,
    -5.24609375,
    -4.60546875,
    -4.57421875,
    -2.796875,
    -1.009765625,
    -3.453125,
    -2.677734375,
    -1.755859375,
    -1.7314453125,
    -1.9384765625,
    -2.849609375,
    -0.51953125,
    -1.7626953125,
    -1.9638671875,
    -1.619140625,
    -0.9111328125,
    -2.890625,
    -1.7265625,
    -2.689453125,
    -2.42578125,
    -3.697265625,
    -1.89453125,
    -2.3203125,
    -2.525390625,
    -2.349609375,
    -3.04296875,
    -0.7236328125,
    -1.9775390625,
    -3.673828125,
    -3.4765625,
    -3.0078125,
    -2.248046875,
    -3.279296875,
    -2.421875,
    -2.75390625,
    -2.107421875,
    -2.51171875,
    -2.03515625,
    -1.369140625,
    -1.447265625,
    -2.24609375,
    -2.763671875,
    -0.5576171875,
    -1.8544921875,
    -2.568359375,
    -1.59375,
    -1.787109375,
    -1.84375,
    -2.05859375,
    -1.7646484375,
    -2.228515625,
    -1.03515625,
    -1.1591796875,
    -1.9267578125,
    -1.5283203125,
    -1.8955078125,
    -2.4453125,
    -1.220703125,
    -2.16796875,
    -2.474609375,
    0.1622314453125,
    -1.9912109375,
    -1.4736328125,
    -1.1943359375,
    0.053802490234375,
    -0.7080078125,
    -2.861328125,
    -0.84716796875,
    -1.6220703125,
    -1.85546875,
    -1.1220703125,
    -2.02734375,
    -1.302734375,
    -1.767578125,
    -0.701171875,
    -1.0234375,
    -0.1539306640625,
    -0.1746826171875,
    -0.51171875,
    0.98486328125,
    0.7978515625,
    -0.277587890625,
    -0.3583984375,
    -0.61181640625,
    0.59765625,
    -0.69970703125,
    0.327392578125,
    0.7451171875,
    1.1279296875,
    -0.12548828125,
    0.0892333984375,
    -0.72607421875,
    0.29931640625,
    0.7412109375,
    -0.7978515625,
    -0.392578125,
    -0.344970703125,
    0.69970703125,
    0.6630859375,
    -0.595703125,
    -0.68505859375,
    2.224609375,
    -0.268798828125,
    -0.1798095703125,
    -0.280029296875,
    0.2232666015625,
    -0.48779296875,
    -4.2109375
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9922448992729187,
    0.003556065494194627,
    0.0035234084352850914,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0006757338414900005
   ],
   "p": [
    0.9929157495498657,
    0.0035584697034209967,
    0.003525790525600314
   ],
   "p_oracle": [
    0.9929444193840027,
    0.0035478509962558746,
    0.0035077447537332773
   ],
   "abstain": 0.0006757338414900005,
   "abstain_oracle": 0.0006824982701800764,
   "abs_delta_abstain": 6.764428690075874e-06,
   "oracle_margin": 0.9893965721130371,
   "max_abs_delta_p": 2.866983413696289e-05,
   "mean_abs_delta_p": 1.911143772304058e-05,
   "max_abs_delta_raw_logits": 0.028242826461791992,
   "mean_abs_delta_raw_logits": 0.005275252624414861,
   "logits_min": -7.96875,
   "logits_max": 3.5078125,
   "raw_logits_min": -7.96875,
   "raw_logits_max": 3.5078125,
   "raw_logits_absmax": 7.96875,
   "oracle_logits_min": -7.96762752532959,
   "oracle_logits_max": 3.5039243698120117,
   "oracle_raw_logits_absmax": 7.96762752532959,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.40254425001330674,
   "seconds": 0.40254425001330674,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r14-wet",
   "request_id": "r14",
   "question_id": "wet",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 61,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    2.06640625,
    -2.05078125,
    -1.783203125,
    -2.44921875,
    -2.310546875,
    -2.509765625,
    -2.83203125,
    -3.228515625,
    -2.962890625,
    -2.2578125,
    -2.236328125,
    -3.076171875,
    -3.8203125,
    -2.359375,
    -3.36328125,
    -4.00390625,
    -3.978515625,
    -4.64453125,
    -2.48046875,
    -3.08203125,
    -3.359375,
    -1.47265625,
    -4.8203125,
    -5.71875,
    -3.181640625,
    -4.0234375,
    -2.771484375,
    -4.9140625,
    -3.658203125,
    -5.93359375,
    -3.849609375,
    -4.609375,
    -5.7734375,
    -3.994140625,
    -2.814453125,
    -4.51953125,
    -4.90625,
    -5.93359375,
    -4.3359375,
    -3.072265625,
    -6.19140625,
    -4.15234375,
    -4.78125,
    -5.78515625,
    -3.58203125,
    -4.16015625,
    -3.91015625,
    -4.7109375,
    -4.671875,
    -4.09375,
    -5.16015625,
    -4.08984375,
    -3.322265625,
    -5.2265625,
    -2.88671875,
    -3.318359375,
    -3.80078125,
    -4.5859375,
    -4.421875,
    -5.96484375,
    -6.91015625,
    -6.74609375,
    -7.765625,
    -5.55078125,
    -3.564453125,
    -6.9609375,
    -7.234375,
    -2.111328125,
    -7.046875,
    -3.98828125,
    -7.46484375,
    -5.0,
    -2.8515625,
    -5.20703125,
    -3.478515625,
    -5.9453125,
    -4.08203125,
    -6.55078125,
    -3.654296875,
    -5.71875,
    -4.359375,
    -3.669921875,
    -2.771484375,
    -4.21875,
    -3.12890625,
    -4.1640625,
    -2.263671875,
    -3.822265625,
    -3.015625,
    -4.2578125,
    -3.314453125,
    -3.83203125,
    -3.8203125,
    -3.31640625,
    -4.27734375,
    -4.16796875,
    -4.6015625,
    -3.16015625,
    -2.6796875,
    -4.08984375,
    -5.23828125,
    -4.67578125,
    -3.111328125,
    -3.640625,
    -4.3046875,
    -5.52734375,
    -3.34765625,
    -4.85546875,
    -3.99609375,
    -5.6484375,
    -4.46875,
    -4.6171875,
    -4.53515625,
    -5.5625,
    -4.5234375,
    -3.67578125,
    -4.4921875,
    -5.43359375,
    -3.79296875,
    -3.48046875,
    -3.009765625,
    -4.12890625,
    -3.03125,
    -3.564453125,
    -2.0390625,
    -4.99609375,
    -5.30859375,
    -2.6484375,
    -2.775390625,
    -5.15625,
    -5.703125,
    -2.669921875,
    -5.3984375,
    -6.046875,
    -4.96875,
    -4.08203125,
    -3.52734375,
    -4.65625,
    -5.57421875,
    -2.478515625,
    -3.25390625,
    -3.986328125,
    -3.244140625,
    -2.537109375,
    -4.62890625,
    -5.109375,
    -5.37890625,
    -5.25,
    -5.11328125,
    -4.2421875,
    -4.1640625,
    -2.603515625,
    -0.87353515625,
    -3.173828125,
    -2.65234375,
    -1.259765625,
    -1.7919921875,
    -1.73046875,
    -2.80859375,
    -0.6201171875,
    -1.6005859375,
    -1.962890625,
    -1.3486328125,
    -1.091796875,
    -2.759765625,
    -1.75,
    -2.458984375,
    -2.1484375,
    -3.611328125,
    -1.9873046875,
    -1.9794921875,
    -2.302734375,
    -2.08203125,
    -2.82421875,
    -0.62841796875,
    -1.755859375,
    -3.537109375,
    -3.076171875,
    -2.65234375,
    -2.13671875,
    -3.287109375,
    -2.21484375,
    -2.640625,
    -1.88671875,
    -2.33984375,
    -1.7890625,
    -1.4248046875,
    -1.400390625,
    -2.03125,
    -2.1953125,
    -0.416748046875,
    -1.6298828125,
    -2.51953125,
    -1.25390625,
    -1.3974609375,
    -1.6650390625,
    -1.9384765625,
    -1.4814453125,
    -1.8486328125,
    -0.85693359375,
    -0.93115234375,
    -1.76953125,
    -1.404296875,
    -1.6318359375,
    -2.08203125,
    -0.8349609375,
    -2.171875,
    -2.306640625,
    0.0157318115234375,
    -1.6953125,
    -1.1591796875,
    -0.92919921875,
    -0.032745361328125,
    -0.548828125,
    -2.56640625,
    -0.84228515625,
    -1.5224609375,
    -1.8583984375,
    -1.1416015625,
    -1.5947265625,
    -0.9013671875,
    -1.55859375,
    -0.90234375,
    -1.0244140625,
    -0.15380859375,
    -0.09600830078125,
    -0.58642578125,
    1.0185546875,
    0.859375,
    -0.1588134765625,
    -0.251220703125,
    -0.339111328125,
    0.26220703125,
    -0.97265625,
    0.451171875,
    0.72412109375,
    1.4091796875,
    -0.415771484375,
    -0.01357269287109375,
    -0.8046875,
    0.357421875,
    0.70263671875,
    -0.7939453125,
    -0.16162109375,
    -0.386474609375,
    0.71142578125,
    0.422607421875,
    -0.7060546875,
    -0.603515625,
    2.05859375,
    -0.2218017578125,
    -0.294921875,
    -0.3876953125,
    0.35498046875,
    -0.489013671875,
    -5.859375
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9831599593162537,
    0.016465364024043083,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003746605943888426
   ],
   "p": [
    0.9835284352302551,
    0.016471534967422485
   ],
   "p_oracle": [
    0.9837594032287598,
    0.01624060794711113
   ],
   "abstain": 0.0003746605943888426,
   "abstain_oracle": 0.00037471865653060377,
   "abs_delta_abstain": 5.806214176118374e-08,
   "oracle_margin": 0.9675188064575195,
   "max_abs_delta_p": 0.00023096799850463867,
   "mean_abs_delta_p": 0.00023094750940799713,
   "max_abs_delta_raw_logits": 0.03394126892089844,
   "mean_abs_delta_raw_logits": 0.0050138380320277065,
   "logits_min": -7.765625,
   "logits_max": 2.06640625,
   "raw_logits_min": -7.765625,
   "raw_logits_max": 2.06640625,
   "raw_logits_absmax": 7.765625,
   "oracle_logits_min": -7.764531135559082,
   "oracle_logits_max": 2.0756096839904785,
   "oracle_raw_logits_absmax": 7.764531135559082,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.34277145797386765,
   "seconds": 0.34277145797386765,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r14-fill",
   "request_id": "r14",
   "question_id": "fill",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 87,
   "nopts": 3,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    2.587890625,
    1.7802734375,
    -2.447265625,
    -2.8515625,
    -2.482421875,
    -3.21875,
    -3.0078125,
    -3.83203125,
    -4.07421875,
    -2.71875,
    -2.68359375,
    -3.78125,
    -4.5078125,
    -2.84765625,
    -3.5703125,
    -4.08984375,
    -3.8359375,
    -4.84375,
    -2.73828125,
    -3.373046875,
    -3.19921875,
    -1.6328125,
    -4.94140625,
    -6.13671875,
    -3.1875,
    -4.50390625,
    -2.16796875,
    -4.93359375,
    -4.03515625,
    -6.49609375,
    -3.9296875,
    -4.53515625,
    -5.6484375,
    -4.61328125,
    -3.060546875,
    -4.921875,
    -5.0,
    -5.66796875,
    -4.140625,
    -2.947265625,
    -6.69140625,
    -4.43359375,
    -4.9375,
    -5.53125,
    -3.7578125,
    -4.671875,
    -3.810546875,
    -4.421875,
    -4.95703125,
    -4.41015625,
    -5.359375,
    -4.38671875,
    -3.005859375,
    -4.83203125,
    -3.021484375,
    -3.44921875,
    -3.841796875,
    -4.3828125,
    -4.359375,
    -5.81640625,
    -7.26171875,
    -6.5078125,
    -7.8828125,
    -5.71875,
    -3.6484375,
    -6.98046875,
    -7.2734375,
    -2.19140625,
    -7.34765625,
    -4.05859375,
    -7.65234375,
    -4.80078125,
    -2.78515625,
    -5.4140625,
    -3.669921875,
    -5.8203125,
    -4.125,
    -6.56640625,
    -3.609375,
    -5.4375,
    -4.08203125,
    -3.69140625,
    -2.52734375,
    -4.23828125,
    -3.2578125,
    -3.8984375,
    -2.369140625,
    -3.681640625,
    -2.91796875,
    -4.53125,
    -3.296875,
    -3.92578125,
    -3.736328125,
    -3.6640625,
    -4.06640625,
    -4.05859375,
    -4.796875,
    -3.54296875,
    -2.998046875,
    -4.00390625,
    -5.4375,
    -4.7578125,
    -3.171875,
    -3.90625,
    -4.203125,
    -5.78125,
    -3.7890625,
    -5.16796875,
    -3.9375,
    -5.578125,
    -4.265625,
    -4.703125,
    -4.7734375,
    -6.12109375,
    -4.734375,
    -3.779296875,
    -4.62109375,
    -5.4921875,
    -3.884765625,
    -3.484375,
    -3.365234375,
    -4.04296875,
    -3.1328125,
    -3.537109375,
    -2.421875,
    -5.20703125,
    -5.4140625,
    -2.681640625,
    -2.650390625,
    -5.1171875,
    -5.40234375,
    -3.03515625,
    -5.27734375,
    -5.85546875,
    -4.96484375,
    -4.1640625,
    -3.5390625,
    -4.88671875,
    -5.6328125,
    -2.849609375,
    -3.541015625,
    -4.08984375,
    -3.16015625,
    -2.685546875,
    -4.80078125,
    -5.12890625,
    -5.78125,
    -4.9921875,
    -4.7421875,
    -4.21484375,
    -4.32421875,
    -2.732421875,
    -1.0185546875,
    -3.083984375,
    -2.603515625,
    -1.369140625,
    -1.794921875,
    -1.9111328125,
    -2.978515625,
    -0.724609375,
    -1.830078125,
    -2.001953125,
    -1.544921875,
    -1.1162109375,
    -2.578125,
    -1.939453125,
    -2.67578125,
    -2.1328125,
    -3.4609375,
    -2.02734375,
    -2.10546875,
    -2.490234375,
    -2.171875,
    -2.9296875,
    -0.79150390625,
    -2.10546875,
    -3.580078125,
    -3.380859375,
    -2.912109375,
    -2.5859375,
    -3.234375,
    -2.65625,
    -2.494140625,
    -2.14453125,
    -2.193359375,
    -1.7392578125,
    -1.37890625,
    -1.4892578125,
    -2.36328125,
    -2.259765625,
    -0.55810546875,
    -1.630859375,
    -2.59765625,
    -1.3681640625,
    -1.455078125,
    -1.5234375,
    -2.0625,
    -1.5927734375,
    -1.84765625,
    -1.0029296875,
    -0.96728515625,
    -1.7578125,
    -1.39453125,
    -1.55859375,
    -2.265625,
    -0.9453125,
    -2.470703125,
    -2.638671875,
    -0.37890625,
    -2.23828125,
    -1.1455078125,
    -1.2490234375,
    -0.21435546875,
    -1.001953125,
    -2.470703125,
    -1.060546875,
    -1.623046875,
    -2.15625,
    -1.2197265625,
    -1.9501953125,
    -1.361328125,
    -1.7919921875,
    -1.0888671875,
    -0.912109375,
    -0.25732421875,
    0.0987548828125,
    -0.5361328125,
    1.103515625,
    0.8486328125,
    -0.2958984375,
    -0.471923828125,
    -0.74853515625,
    0.08428955078125,
    -0.900390625,
    0.11810302734375,
    0.90234375,
    1.3935546875,
    -0.311767578125,
    0.2464599609375,
    -0.8466796875,
    0.355712890625,
    0.59033203125,
    -0.54736328125,
    -0.148193359375,
    -0.0977783203125,
    0.46728515625,
    0.4794921875,
    -0.2437744140625,
    -0.90185546875,
    2.015625,
    -0.361328125,
    -0.4443359375,
    -0.204345703125,
    0.3369140625,
    -0.57763671875,
    -5.2890625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.6804565787315369,
    0.31373459100723267,
    0.005451255012303591,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003575703885871917
   ],
   "p": [
    0.6807000041007996,
    0.3138468265533447,
    0.005453204736113548
   ],
   "p_oracle": [
    0.675521969795227,
    0.3190520703792572,
    0.005425968673080206
   ],
   "abstain": 0.0003575703885871917,
   "abstain_oracle": 0.000354081392288208,
   "abs_delta_abstain": 3.488996298983693e-06,
   "oracle_margin": 0.35646989941596985,
   "max_abs_delta_p": 0.005205243825912476,
   "mean_abs_delta_p": 0.0034701713981727758,
   "max_abs_delta_raw_logits": 0.026024818420410156,
   "mean_abs_delta_raw_logits": 0.004977937787771225,
   "logits_min": -7.8828125,
   "logits_max": 2.587890625,
   "raw_logits_min": -7.8828125,
   "raw_logits_max": 2.587890625,
   "raw_logits_absmax": 7.8828125,
   "oracle_logits_min": -7.882862567901611,
   "oracle_logits_max": 2.586092710494995,
   "oracle_raw_logits_absmax": 7.882862567901611,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4891923750983551,
   "seconds": 0.4891923750983551,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r15-color",
   "request_id": "r15",
   "question_id": "color",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 53,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.75,
    3.36328125,
    -2.76171875,
    -2.9375,
    -2.298828125,
    -3.20703125,
    -3.1015625,
    -3.7890625,
    -3.92578125,
    -2.78515625,
    -3.1953125,
    -3.931640625,
    -5.5078125,
    -3.716796875,
    -4.4453125,
    -4.62109375,
    -3.912109375,
    -4.1640625,
    -3.181640625,
    -3.251953125,
    -3.638671875,
    -2.271484375,
    -5.484375,
    -7.26953125,
    -4.18359375,
    -5.78515625,
    -2.744140625,
    -5.546875,
    -4.62890625,
    -6.890625,
    -4.7734375,
    -4.81640625,
    -6.42578125,
    -5.22265625,
    -3.341796875,
    -5.703125,
    -5.48046875,
    -5.91796875,
    -4.44140625,
    -3.16796875,
    -7.4296875,
    -4.6796875,
    -4.86328125,
    -6.2890625,
    -4.609375,
    -5.36328125,
    -4.5546875,
    -4.55859375,
    -5.33984375,
    -4.921875,
    -5.859375,
    -4.734375,
    -3.037109375,
    -5.05859375,
    -2.9140625,
    -3.39453125,
    -3.435546875,
    -4.50390625,
    -4.79296875,
    -5.6484375,
    -7.73046875,
    -6.61328125,
    -7.796875,
    -6.80078125,
    -3.783203125,
    -7.7421875,
    -7.875,
    -2.353515625,
    -8.1171875,
    -4.296875,
    -8.1875,
    -5.62890625,
    -2.658203125,
    -6.3125,
    -3.869140625,
    -6.5234375,
    -4.453125,
    -7.1015625,
    -4.36328125,
    -5.90234375,
    -4.26953125,
    -4.34765625,
    -2.818359375,
    -4.45703125,
    -3.81640625,
    -4.171875,
    -2.70703125,
    -4.15625,
    -3.0546875,
    -5.2578125,
    -3.681640625,
    -4.18359375,
    -4.421875,
    -4.40625,
    -4.75,
    -4.93359375,
    -5.25390625,
    -3.923828125,
    -4.27734375,
    -4.23828125,
    -6.4609375,
    -4.6796875,
    -3.76953125,
    -4.3984375,
    -4.68359375,
    -6.37890625,
    -4.515625,
    -5.92578125,
    -4.33984375,
    -5.86328125,
    -4.59375,
    -4.9609375,
    -5.4375,
    -6.734375,
    -4.984375,
    -4.1796875,
    -4.7265625,
    -6.34375,
    -4.6015625,
    -3.740234375,
    -3.509765625,
    -4.01171875,
    -3.40234375,
    -3.919921875,
    -2.591796875,
    -6.02734375,
    -5.98828125,
    -3.337890625,
    -3.33203125,
    -5.703125,
    -5.89453125,
    -3.8984375,
    -5.69140625,
    -6.1640625,
    -5.20703125,
    -5.0234375,
    -4.2421875,
    -5.26171875,
    -6.0625,
    -3.193359375,
    -3.83203125,
    -4.30078125,
    -3.89453125,
    -3.494140625,
    -4.84375,
    -5.53515625,
    -6.390625,
    -5.2890625,
    -4.8125,
    -4.65234375,
    -4.9140625,
    -3.302734375,
    -1.291015625,
    -3.603515625,
    -2.439453125,
    -1.8173828125,
    -2.056640625,
    -2.087890625,
    -3.478515625,
    -0.47998046875,
    -2.185546875,
    -2.123046875,
    -1.7265625,
    -1.306640625,
    -2.638671875,
    -2.400390625,
    -2.978515625,
    -1.8251953125,
    -3.556640625,
    -2.1796875,
    -2.609375,
    -2.48828125,
    -2.58984375,
    -3.169921875,
    -0.93798828125,
    -2.078125,
    -3.3203125,
    -3.369140625,
    -3.21875,
    -3.755859375,
    -3.580078125,
    -3.439453125,
    -2.845703125,
    -2.318359375,
    -2.75390625,
    -1.7998046875,
    -1.310546875,
    -1.9677734375,
    -2.185546875,
    -2.263671875,
    -1.216796875,
    -1.8798828125,
    -3.177734375,
    -1.8759765625,
    -1.619140625,
    -2.298828125,
    -2.099609375,
    -2.115234375,
    -1.7958984375,
    -1.67578125,
    -1.443359375,
    -2.109375,
    -1.7744140625,
    -2.04296875,
    -2.54296875,
    -1.3369140625,
    -2.794921875,
    -2.736328125,
    -0.42333984375,
    -2.42578125,
    -1.017578125,
    -1.3876953125,
    -0.99853515625,
    -1.740234375,
    -2.529296875,
    -1.255859375,
    -1.8017578125,
    -2.51171875,
    -2.11328125,
    -2.177734375,
    -1.5380859375,
    -1.673828125,
    -1.5078125,
    -1.3046875,
    -0.21337890625,
    0.07073974609375,
    -0.40625,
    0.8271484375,
    1.1318359375,
    -0.0606689453125,
    -0.78515625,
    -0.53173828125,
    0.1702880859375,
    -0.77197265625,
    0.5029296875,
    1.1494140625,
    1.31640625,
    -0.68994140625,
    -0.046356201171875,
    -0.75634765625,
    0.1951904296875,
    0.8349609375,
    -0.95458984375,
    -0.10870361328125,
    0.2012939453125,
    0.56640625,
    0.382568359375,
    -0.250244140625,
    -0.97265625,
    1.341796875,
    -0.047027587890625,
    -0.73095703125,
    -0.296142578125,
    0.4423828125,
    -0.63330078125,
    -3.94140625
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0030813461635261774,
    0.9928714632987976,
    0.0030474215745925903,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.000999841489829123
   ],
   "p": [
    0.003084430005401373,
    0.9938651323318481,
    0.0030504714231938124
   ],
   "p_oracle": [
    0.0030322920065373182,
    0.9939413070678711,
    0.0030264435335993767
   ],
   "abstain": 0.000999841489829123,
   "abstain_oracle": 0.001004166784696281,
   "abs_delta_abstain": 4.325294867157936e-06,
   "oracle_margin": 0.9909090399742126,
   "max_abs_delta_p": 7.617473602294922e-05,
   "mean_abs_delta_p": 5.0780208160479866e-05,
   "max_abs_delta_raw_logits": 0.03212738037109375,
   "mean_abs_delta_raw_logits": 0.005439665372250602,
   "logits_min": -8.1875,
   "logits_max": 3.36328125,
   "raw_logits_min": -8.1875,
   "raw_logits_max": 3.36328125,
   "raw_logits_absmax": 8.1875,
   "oracle_logits_min": -8.178394317626953,
   "oracle_logits_max": 3.368619918823242,
   "oracle_raw_logits_absmax": 8.178394317626953,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2995249159866944,
   "seconds": 0.2995249159866944,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r15-level",
   "request_id": "r15",
   "question_id": "level",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 125,
   "nopts": 10,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    -1.626953125,
    -2.201171875,
    -2.5078125,
    -2.537109375,
    -2.3359375,
    -3.474609375,
    -2.912109375,
    4.90234375,
    -2.3828125,
    -0.6640625,
    -2.935546875,
    -2.8515625,
    -4.6953125,
    -3.287109375,
    -3.861328125,
    -3.943359375,
    -3.521484375,
    -4.765625,
    -2.994140625,
    -2.619140625,
    -3.701171875,
    -1.744140625,
    -4.140625,
    -4.359375,
    -3.359375,
    -4.3203125,
    -3.181640625,
    -5.15234375,
    -4.7109375,
    -5.9921875,
    -4.375,
    -5.06640625,
    -6.0234375,
    -4.8671875,
    -3.0,
    -4.48828125,
    -5.3203125,
    -5.73828125,
    -4.3828125,
    -2.599609375,
    -6.4609375,
    -3.677734375,
    -4.78125,
    -6.3984375,
    -4.0546875,
    -4.0546875,
    -4.21484375,
    -5.16796875,
    -4.7421875,
    -4.234375,
    -5.5,
    -4.03125,
    -2.80078125,
    -5.08984375,
    -3.419921875,
    -3.314453125,
    -4.22265625,
    -4.76953125,
    -4.390625,
    -5.421875,
    -7.4375,
    -6.9453125,
    -7.8203125,
    -5.69140625,
    -3.75390625,
    -7.3984375,
    -6.78515625,
    -1.91796875,
    -7.359375,
    -4.20703125,
    -7.75390625,
    -5.53125,
    -2.4921875,
    -5.671875,
    -3.421875,
    -6.375,
    -4.41015625,
    -6.71484375,
    -3.751953125,
    -5.61328125,
    -4.515625,
    -4.34765625,
    -2.392578125,
    -4.515625,
    -3.076171875,
    -4.859375,
    -2.294921875,
    -3.87890625,
    -2.60546875,
    -3.97265625,
    -3.419921875,
    -3.48828125,
    -4.09375,
    -3.400390625,
    -4.85546875,
    -4.296875,
    -5.35546875,
    -3.216796875,
    -3.228515625,
    -4.50390625,
    -5.7265625,
    -4.49609375,
    -3.85546875,
    -3.86328125,
    -4.55078125,
    -5.62109375,
    -3.609375,
    -5.265625,
    -4.07421875,
    -5.72265625,
    -4.42578125,
    -4.71875,
    -4.56640625,
    -5.6171875,
    -5.08203125,
    -3.962890625,
    -4.5625,
    -6.625,
    -4.15625,
    -3.240234375,
    -3.1875,
    -3.71875,
    -3.21875,
    -3.384765625,
    -2.109375,
    -4.96484375,
    -5.21875,
    -2.515625,
    -2.9453125,
    -5.46484375,
    -5.8515625,
    -2.46875,
    -5.3515625,
    -5.703125,
    -4.96875,
    -4.6015625,
    -3.556640625,
    -4.8125,
    -5.90234375,
    -2.267578125,
    -2.9453125,
    -3.67578125,
    -3.525390625,
    -2.517578125,
    -4.65234375,
    -4.8671875,
    -6.0,
    -5.08203125,
    -5.54296875,
    -5.41796875,
    -4.140625,
    -2.9765625,
    -0.91796875,
    -3.607421875,
    -2.466796875,
    -1.412109375,
    -1.513671875,
    -2.005859375,
    -3.2265625,
    -0.0557861328125,
    -1.3701171875,
    -2.19921875,
    -1.732421875,
    -1.412109375,
    -3.21484375,
    -1.2451171875,
    -3.353515625,
    -2.517578125,
    -3.6171875,
    -2.5,
    -1.8486328125,
    -2.583984375,
    -2.009765625,
    -3.16015625,
    -0.2044677734375,
    -1.7548828125,
    -3.666015625,
    -3.23046875,
    -3.267578125,
    -2.28515625,
    -3.67578125,
    -1.955078125,
    -2.619140625,
    -1.599609375,
    -2.392578125,
    -1.7001953125,
    -0.9697265625,
    -2.08984375,
    -1.833984375,
    -2.0625,
    -1.1220703125,
    -1.5322265625,
    -2.482421875,
    -1.818359375,
    -1.7275390625,
    -1.626953125,
    -1.341796875,
    -1.833984375,
    -1.833984375,
    -1.001953125,
    -1.3056640625,
    -2.177734375,
    -1.5791015625,
    -1.8798828125,
    -2.248046875,
    -0.95166015625,
    -2.70703125,
    -2.5703125,
    0.2364501953125,
    -2.37109375,
    -1.3955078125,
    -1.7841796875,
    -0.2418212890625,
    -1.30078125,
    -2.328125,
    -1.1103515625,
    -1.548828125,
    -1.966796875,
    -0.74658203125,
    -1.681640625,
    -1.439453125,
    -1.130859375,
    -1.0009765625,
    -1.0361328125,
    -0.160400390625,
    -0.40869140625,
    -0.47998046875,
    1.2275390625,
    1.103515625,
    -0.39794921875,
    -0.268310546875,
    -0.7265625,
    0.422119140625,
    -0.29541015625,
    0.273681640625,
    1.0986328125,
    1.251953125,
    -0.346435546875,
    -0.40234375,
    -0.765625,
    -0.326171875,
    0.8271484375,
    -0.84130859375,
    -0.1868896484375,
    -0.059326171875,
    0.73388671875,
    0.115234375,
    -0.20703125,
    -0.4462890625,
    1.9697265625,
    0.194091796875,
    0.042572021484375,
    -0.6982421875,
    0.233642578125,
    -0.728515625,
    -5.44140625
   ],
   "raw256_argmax": 7,
   "option_argmax": 7,
   "oracle_argmax": 7,
   "argmax_agrees": true,
   "p_full": [
    0.0018894472159445286,
    0.0010896030580624938,
    0.0008120887214317918,
    0.0007895987364463508,
    0.0009575489675626159,
    0.0003214353637304157,
    0.0005511646158993244,
    0.9878690838813782,
    0.0009154723957180977,
    0.004755742382258177,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    4.8780973884277046e-05
   ],
   "p": [
    0.0018895394168794155,
    0.0010896562598645687,
    0.0008121283608488739,
    0.0007896372699178755,
    0.0009575957083143294,
    0.0003214510506950319,
    0.0005511915078386664,
    0.9879173040390015,
    0.0009155170992016792,
    0.004755974747240543
   ],
   "p_oracle": [
    0.0018778711091727018,
    0.001082923263311386,
    0.0007980354130268097,
    0.0007786157075315714,
    0.0009413186344318092,
    0.00032714440021663904,
    0.0005481148837134242,
    0.9876933097839355,
    0.0009699876536615193,
    0.004982673097401857
   ],
   "abstain": 4.8780973884277046e-05,
   "abstain_oracle": 4.8452511691721156e-05,
   "abs_delta_abstain": 3.284621925558895e-07,
   "oracle_margin": 0.9827106595039368,
   "max_abs_delta_p": 0.000226698350161314,
   "mean_abs_delta_p": 5.7372602168470624e-05,
   "max_abs_delta_raw_logits": 0.05943560600280762,
   "mean_abs_delta_raw_logits": 0.006008546275552362,
   "logits_min": -7.8203125,
   "logits_max": 4.90234375,
   "raw_logits_min": -7.8203125,
   "raw_logits_max": 4.90234375,
   "raw_logits_absmax": 7.8203125,
   "oracle_logits_min": -7.816807746887207,
   "oracle_logits_max": 4.9012556076049805,
   "oracle_raw_logits_absmax": 7.816807746887207,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.7018402909161523,
   "seconds": 0.7018402909161523,
   "full_vocab_argmax_id": 7,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r15-position",
   "request_id": "r15",
   "question_id": "position",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 59,
   "nopts": 2,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    2.314453125,
    -0.677734375,
    -1.958984375,
    -2.48828125,
    -1.7802734375,
    -2.712890625,
    -2.482421875,
    -3.294921875,
    -3.490234375,
    -1.8046875,
    -2.169921875,
    -3.009765625,
    -3.849609375,
    -2.16015625,
    -3.212890625,
    -3.75,
    -3.869140625,
    -4.9921875,
    -2.7578125,
    -2.845703125,
    -3.162109375,
    -1.5419921875,
    -4.80078125,
    -5.7578125,
    -3.0390625,
    -3.7890625,
    -2.080078125,
    -4.203125,
    -3.611328125,
    -5.74609375,
    -3.732421875,
    -3.806640625,
    -5.4140625,
    -3.9296875,
    -2.658203125,
    -4.25390625,
    -4.7890625,
    -5.765625,
    -3.947265625,
    -2.83203125,
    -6.30859375,
    -3.982421875,
    -4.62890625,
    -5.59765625,
    -3.353515625,
    -4.21484375,
    -3.6484375,
    -4.42578125,
    -4.82421875,
    -3.92578125,
    -5.11328125,
    -4.16015625,
    -2.861328125,
    -5.02734375,
    -2.794921875,
    -3.291015625,
    -3.810546875,
    -4.53125,
    -4.24609375,
    -5.8359375,
    -6.8125,
    -6.2734375,
    -7.65625,
    -5.48046875,
    -3.462890625,
    -6.8046875,
    -6.97265625,
    -2.3125,
    -7.109375,
    -3.99609375,
    -7.34765625,
    -4.70703125,
    -2.611328125,
    -5.0546875,
    -3.4765625,
    -5.49609375,
    -3.822265625,
    -6.4296875,
    -3.427734375,
    -5.23046875,
    -4.12890625,
    -3.6328125,
    -2.384765625,
    -4.109375,
    -3.138671875,
    -4.00390625,
    -2.123046875,
    -3.634765625,
    -2.921875,
    -4.109375,
    -3.15234375,
    -3.83984375,
    -3.623046875,
    -3.357421875,
    -4.12890625,
    -3.984375,
    -4.5390625,
    -3.19140625,
    -2.705078125,
    -3.89453125,
    -4.91796875,
    -4.45703125,
    -3.111328125,
    -3.640625,
    -4.12109375,
    -5.2265625,
    -3.248046875,
    -4.72265625,
    -3.75,
    -5.37890625,
    -4.03515625,
    -4.27734375,
    -4.35546875,
    -5.46875,
    -4.453125,
    -3.46875,
    -4.41796875,
    -5.21484375,
    -3.451171875,
    -3.203125,
    -2.978515625,
    -4.11328125,
    -2.88671875,
    -3.3828125,
    -2.048828125,
    -4.7890625,
    -5.11328125,
    -2.37890625,
    -2.466796875,
    -4.82421875,
    -5.34765625,
    -2.421875,
    -5.10546875,
    -5.66796875,
    -4.85546875,
    -3.6796875,
    -3.201171875,
    -4.3984375,
    -5.4453125,
    -2.396484375,
    -3.25,
    -3.853515625,
    -2.908203125,
    -2.458984375,
    -4.55859375,
    -4.66796875,
    -5.171875,
    -4.9765625,
    -4.84375,
    -4.0546875,
    -4.17578125,
    -2.376953125,
    -0.55712890625,
    -2.931640625,
    -2.3828125,
    -1.140625,
    -1.474609375,
    -1.7958984375,
    -2.7421875,
    -0.44482421875,
    -1.6630859375,
    -1.876953125,
    -1.2861328125,
    -0.880859375,
    -2.52734375,
    -1.748046875,
    -2.34765625,
    -2.037109375,
    -3.322265625,
    -1.765625,
    -1.779296875,
    -2.2578125,
    -1.912109375,
    -2.73046875,
    -0.365478515625,
    -1.5654296875,
    -3.234375,
    -3.064453125,
    -2.501953125,
    -1.9697265625,
    -2.943359375,
    -1.896484375,
    -2.365234375,
    -1.8515625,
    -2.0625,
    -1.6259765625,
    -1.244140625,
    -1.2939453125,
    -2.109375,
    -2.015625,
    -0.08770751953125,
    -1.50390625,
    -2.2109375,
    -1.27734375,
    -1.2822265625,
    -1.396484375,
    -1.931640625,
    -1.3466796875,
    -1.7138671875,
    -0.8154296875,
    -0.82958984375,
    -1.46484375,
    -1.171875,
    -1.404296875,
    -2.0546875,
    -0.62353515625,
    -2.1328125,
    -2.134765625,
    -0.08453369140625,
    -1.705078125,
    -0.88330078125,
    -0.93603515625,
    0.1766357421875,
    -0.47412109375,
    -2.4765625,
    -0.72314453125,
    -1.4228515625,
    -1.86328125,
    -0.99755859375,
    -1.6787109375,
    -1.171875,
    -1.74609375,
    -0.91552734375,
    -0.90576171875,
    -0.15380859375,
    0.08514404296875,
    -0.50146484375,
    1.171875,
    0.83837890625,
    -0.167236328125,
    -0.419189453125,
    -0.7275390625,
    0.335205078125,
    -0.845703125,
    0.281982421875,
    0.716796875,
    1.3681640625,
    -0.1502685546875,
    -0.06304931640625,
    -0.8251953125,
    0.46728515625,
    0.71484375,
    -0.591796875,
    -0.1475830078125,
    -0.1766357421875,
    0.5146484375,
    0.55126953125,
    -0.55029296875,
    -0.84765625,
    2.126953125,
    -0.457275390625,
    -0.423828125,
    -0.257080078125,
    0.343505859375,
    -0.461669921875,
    -5.90625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.945925772190094,
    0.05371670052409172,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003575224836822599
   ],
   "p": [
    0.9462640881538391,
    0.05373591184616089
   ],
   "p_oracle": [
    0.9450725317001343,
    0.05492749810218811
   ],
   "abstain": 0.0003575224836822599,
   "abstain_oracle": 0.00036147201899439096,
   "abs_delta_abstain": 3.949535312131047e-06,
   "oracle_margin": 0.8901450634002686,
   "max_abs_delta_p": 0.0011915862560272217,
   "mean_abs_delta_p": 0.0011915713548660278,
   "max_abs_delta_raw_logits": 0.02574944496154785,
   "mean_abs_delta_raw_logits": 0.006210377672687173,
   "logits_min": -7.65625,
   "logits_max": 2.314453125,
   "raw_logits_min": -7.65625,
   "raw_logits_max": 2.314453125,
   "raw_logits_absmax": 7.65625,
   "oracle_logits_min": -7.648866653442383,
   "oracle_logits_max": 2.306751251220703,
   "oracle_raw_logits_absmax": 7.648866653442383,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.32851070794276893,
   "seconds": 0.32851070794276893,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r16-color",
   "request_id": "r16",
   "question_id": "color",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 71,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.3671875,
    3.09375,
    -2.83203125,
    -3.130859375,
    -2.537109375,
    -2.982421875,
    -3.01953125,
    -4.02734375,
    -3.85546875,
    -2.896484375,
    -3.279296875,
    -4.08203125,
    -5.375,
    -3.478515625,
    -4.31640625,
    -4.4921875,
    -3.724609375,
    -4.1640625,
    -3.19140625,
    -3.310546875,
    -3.5390625,
    -2.216796875,
    -5.359375,
    -7.1171875,
    -3.935546875,
    -5.44140625,
    -2.521484375,
    -5.32421875,
    -4.58984375,
    -6.67578125,
    -4.68359375,
    -4.67578125,
    -6.3125,
    -5.1484375,
    -3.482421875,
    -5.60546875,
    -5.40625,
    -5.9765625,
    -4.375,
    -3.0390625,
    -7.16015625,
    -4.58203125,
    -4.76953125,
    -5.98828125,
    -4.578125,
    -5.265625,
    -4.5,
    -4.421875,
    -5.3828125,
    -4.6796875,
    -5.7421875,
    -4.80078125,
    -2.87890625,
    -4.90234375,
    -2.931640625,
    -3.44140625,
    -3.578125,
    -4.32421875,
    -4.6875,
    -5.3203125,
    -7.421875,
    -6.41796875,
    -7.57421875,
    -6.6484375,
    -3.78125,
    -7.53125,
    -7.7734375,
    -2.412109375,
    -7.94140625,
    -4.12109375,
    -7.8125,
    -5.390625,
    -2.56640625,
    -6.29296875,
    -3.7421875,
    -6.3203125,
    -4.33203125,
    -6.953125,
    -4.3828125,
    -5.78515625,
    -4.39453125,
    -4.29296875,
    -2.765625,
    -4.45703125,
    -3.677734375,
    -4.0703125,
    -2.619140625,
    -3.9765625,
    -3.15234375,
    -5.078125,
    -3.568359375,
    -4.18359375,
    -4.44140625,
    -4.2421875,
    -4.59765625,
    -4.76953125,
    -5.14453125,
    -4.04296875,
    -4.0703125,
    -4.15234375,
    -6.2734375,
    -4.50390625,
    -3.732421875,
    -4.265625,
    -4.4921875,
    -6.30078125,
    -4.43359375,
    -5.796875,
    -4.1484375,
    -5.734375,
    -4.47265625,
    -4.84765625,
    -5.24609375,
    -6.45703125,
    -4.86328125,
    -4.02734375,
    -4.65625,
    -6.28125,
    -4.6015625,
    -3.572265625,
    -3.357421875,
    -3.923828125,
    -3.279296875,
    -3.82421875,
    -2.345703125,
    -5.98828125,
    -5.91796875,
    -3.20703125,
    -3.25,
    -5.5859375,
    -5.66015625,
    -3.93359375,
    -5.703125,
    -6.125,
    -4.81640625,
    -4.90234375,
    -3.978515625,
    -4.98046875,
    -5.9921875,
    -3.158203125,
    -3.7265625,
    -4.26953125,
    -3.8203125,
    -3.50390625,
    -4.7734375,
    -5.359375,
    -6.2421875,
    -5.1328125,
    -4.68359375,
    -4.41015625,
    -4.8515625,
    -3.267578125,
    -1.3427734375,
    -3.501953125,
    -2.330078125,
    -1.7607421875,
    -2.025390625,
    -2.005859375,
    -3.470703125,
    -0.51806640625,
    -2.158203125,
    -1.9609375,
    -1.8251953125,
    -1.3681640625,
    -2.48828125,
    -2.294921875,
    -2.880859375,
    -1.8896484375,
    -3.541015625,
    -2.1484375,
    -2.455078125,
    -2.52734375,
    -2.560546875,
    -3.2265625,
    -0.85888671875,
    -2.23828125,
    -3.33203125,
    -3.501953125,
    -3.33203125,
    -3.716796875,
    -3.771484375,
    -3.4921875,
    -2.701171875,
    -2.39453125,
    -2.728515625,
    -1.8017578125,
    -1.40234375,
    -1.8564453125,
    -2.2421875,
    -2.3671875,
    -1.1669921875,
    -1.986328125,
    -3.322265625,
    -1.8974609375,
    -1.509765625,
    -2.267578125,
    -2.1640625,
    -2.1875,
    -1.783203125,
    -1.7177734375,
    -1.484375,
    -2.138671875,
    -1.9287109375,
    -2.130859375,
    -2.48046875,
    -1.3271484375,
    -2.673828125,
    -2.744140625,
    -0.47705078125,
    -2.400390625,
    -1.0908203125,
    -1.6064453125,
    -1.0234375,
    -1.66015625,
    -2.65625,
    -1.4033203125,
    -2.078125,
    -2.4921875,
    -2.025390625,
    -2.12109375,
    -1.5751953125,
    -1.6875,
    -1.4619140625,
    -1.3271484375,
    -0.0645751953125,
    -0.00628662109375,
    -0.481201171875,
    0.7841796875,
    1.03125,
    -0.036865234375,
    -0.73291015625,
    -0.475341796875,
    0.0924072265625,
    -0.61962890625,
    0.4521484375,
    1.197265625,
    1.248046875,
    -0.61572265625,
    0.01099395751953125,
    -0.7060546875,
    0.332763671875,
    0.84228515625,
    -0.79736328125,
    -0.199951171875,
    0.11163330078125,
    0.525390625,
    0.56201171875,
    -0.2109375,
    -0.95654296875,
    1.46875,
    -0.2230224609375,
    -0.6201171875,
    -0.333251953125,
    0.401611328125,
    -0.440673828125,
    -3.181640625
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.005678851623088121,
    0.9880296587944031,
    0.0036605363711714745,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.002630917588248849
   ],
   "p": [
    0.005693831946700811,
    0.9906359910964966,
    0.0036701925564557314
   ],
   "p_oracle": [
    0.005624168086796999,
    0.9907371401786804,
    0.0036387406289577484
   ],
   "abstain": 0.002630917588248849,
   "abstain_oracle": 0.0025959154590964317,
   "abs_delta_abstain": 3.500212915241718e-05,
   "oracle_margin": 0.9851129651069641,
   "max_abs_delta_p": 0.00010114908218383789,
   "mean_abs_delta_p": 6.74216231952111e-05,
   "max_abs_delta_raw_logits": 0.032731056213378906,
   "mean_abs_delta_raw_logits": 0.005183964618481696,
   "logits_min": -7.94140625,
   "logits_max": 3.09375,
   "raw_logits_min": -7.94140625,
   "raw_logits_max": 3.09375,
   "raw_logits_absmax": 7.94140625,
   "oracle_logits_min": -7.9451446533203125,
   "oracle_logits_max": 3.1033008098602295,
   "oracle_raw_logits_absmax": 7.9451446533203125,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.39627779205329716,
   "seconds": 0.39627779205329716,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r16-level",
   "request_id": "r16",
   "question_id": "level",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 160,
   "nopts": 10,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    -1.1787109375,
    -2.443359375,
    -2.10546875,
    3.326171875,
    -1.439453125,
    -2.927734375,
    -3.333984375,
    -3.87109375,
    -4.4609375,
    -1.46484375,
    -1.50390625,
    -2.982421875,
    -3.341796875,
    -1.501953125,
    -3.517578125,
    -4.515625,
    -4.3828125,
    -5.046875,
    -2.677734375,
    -2.7578125,
    -3.75390625,
    -1.6845703125,
    -5.70703125,
    -5.375,
    -3.931640625,
    -4.3046875,
    -2.759765625,
    -4.8125,
    -3.49609375,
    -4.765625,
    -4.21875,
    -4.23046875,
    -6.234375,
    -2.94140625,
    -1.9638671875,
    -4.76953125,
    -4.921875,
    -5.6484375,
    -3.865234375,
    -3.140625,
    -6.6875,
    -4.1015625,
    -5.03515625,
    -5.86328125,
    -3.470703125,
    -4.2265625,
    -4.43359375,
    -4.859375,
    -5.4140625,
    -3.939453125,
    -4.46484375,
    -4.046875,
    -2.85546875,
    -5.63671875,
    -2.328125,
    -2.525390625,
    -4.3203125,
    -4.7265625,
    -4.36328125,
    -5.82421875,
    -7.08203125,
    -6.66015625,
    -7.7265625,
    -5.64453125,
    -3.474609375,
    -6.80859375,
    -7.3671875,
    -1.7509765625,
    -7.3984375,
    -4.46875,
    -7.66015625,
    -5.15625,
    -2.416015625,
    -5.09765625,
    -3.6953125,
    -5.625,
    -4.43359375,
    -6.54296875,
    -3.5,
    -5.2265625,
    -4.19921875,
    -3.599609375,
    -2.345703125,
    -3.94921875,
    -3.087890625,
    -4.1171875,
    -2.556640625,
    -2.986328125,
    -3.267578125,
    -4.30859375,
    -3.646484375,
    -4.09765625,
    -3.8359375,
    -3.4921875,
    -4.54296875,
    -4.61328125,
    -4.78515625,
    -3.337890625,
    -3.384765625,
    -3.732421875,
    -5.06640625,
    -5.203125,
    -3.0390625,
    -3.6875,
    -4.45703125,
    -5.2890625,
    -3.474609375,
    -4.40625,
    -4.37890625,
    -5.5078125,
    -4.26171875,
    -4.75,
    -4.30078125,
    -5.54296875,
    -4.87109375,
    -3.927734375,
    -4.046875,
    -5.4765625,
    -3.712890625,
    -2.83984375,
    -2.69921875,
    -4.1953125,
    -3.146484375,
    -3.71875,
    -1.98828125,
    -5.15234375,
    -5.16796875,
    -2.65625,
    -3.4765625,
    -5.4453125,
    -5.30859375,
    -2.349609375,
    -5.8125,
    -5.9375,
    -4.3984375,
    -4.26171875,
    -3.7890625,
    -4.73046875,
    -5.7265625,
    -2.27734375,
    -3.044921875,
    -4.25,
    -3.111328125,
    -3.05859375,
    -4.0546875,
    -4.625,
    -5.75,
    -5.5234375,
    -5.60546875,
    -4.46875,
    -4.2109375,
    -2.615234375,
    -1.279296875,
    -3.55859375,
    -2.419921875,
    -1.4921875,
    -1.7822265625,
    -1.6796875,
    -2.66796875,
    -1.134765625,
    -2.279296875,
    -2.62890625,
    -1.556640625,
    -1.005859375,
    -3.40234375,
    -1.6904296875,
    -2.560546875,
    -2.263671875,
    -3.953125,
    -2.046875,
    -1.818359375,
    -2.171875,
    -2.177734375,
    -3.244140625,
    -1.0634765625,
    -1.8037109375,
    -3.359375,
    -3.048828125,
    -2.380859375,
    -2.80859375,
    -3.13671875,
    -2.318359375,
    -2.66796875,
    -1.71875,
    -2.818359375,
    -1.9580078125,
    -1.5537109375,
    -2.044921875,
    -2.005859375,
    -1.697265625,
    -0.982421875,
    -1.392578125,
    -2.421875,
    -1.48828125,
    -1.4248046875,
    -2.1484375,
    -1.501953125,
    -1.28515625,
    -1.94921875,
    -0.99560546875,
    -1.611328125,
    -1.5859375,
    -1.2314453125,
    -2.00390625,
    -2.04296875,
    -1.19921875,
    -1.9072265625,
    -2.740234375,
    0.25390625,
    -2.099609375,
    -0.72607421875,
    -1.310546875,
    -0.54541015625,
    -1.08203125,
    -2.330078125,
    -0.97021484375,
    -1.677734375,
    -2.0546875,
    -1.30078125,
    -1.68359375,
    -1.404296875,
    -1.6123046875,
    -1.0830078125,
    -1.4873046875,
    -0.08636474609375,
    0.1998291015625,
    -0.830078125,
    0.80859375,
    1.056640625,
    -0.478271484375,
    -0.927734375,
    -0.5146484375,
    -0.45947265625,
    -1.099609375,
    0.5634765625,
    0.6708984375,
    1.26171875,
    -0.53564453125,
    -0.473388671875,
    -0.487060546875,
    0.69921875,
    0.4296875,
    -0.7705078125,
    0.286865234375,
    -0.0771484375,
    0.09771728515625,
    0.2841796875,
    -0.466796875,
    -0.18408203125,
    1.791015625,
    -0.11688232421875,
    -0.62646484375,
    -0.5634765625,
    1.1103515625,
    -0.1773681640625,
    -5.34375
   ],
   "raw256_argmax": 3,
   "option_argmax": 3,
   "oracle_argmax": 3,
   "argmax_agrees": true,
   "p_full": [
    0.012693560682237148,
    0.003776311641559005,
    0.005220871418714523,
    0.953052282333374,
    0.009886160492897034,
    0.002373594092205167,
    0.001607944373972714,
    0.0009608473628759384,
    0.0005458617233671248,
    0.009648430161178112,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00023417382908519357
   ],
   "p": [
    0.012696533463895321,
    0.003777195932343602,
    0.0052220942452549934,
    0.9532754421234131,
    0.009888475760817528,
    0.0023741498589515686,
    0.001608320977538824,
    0.0009610723936930299,
    0.0005459895473904908,
    0.00965068954974413
   ],
   "p_oracle": [
    0.011903850361704826,
    0.0036331256851553917,
    0.0049773394130170345,
    0.9555612802505493,
    0.009435553103685379,
    0.0023825846146792173,
    0.00155331299174577,
    0.0009360660333186388,
    0.0005382035742513835,
    0.009078679606318474
   ],
   "abstain": 0.00023417382908519357,
   "abstain_oracle": 0.00022859007003717124,
   "abs_delta_abstain": 5.58375904802233e-06,
   "oracle_margin": 0.9436574578285217,
   "max_abs_delta_p": 0.0022858381271362305,
   "mean_abs_delta_p": 0.00045885139843448997,
   "max_abs_delta_raw_logits": 0.04090452194213867,
   "mean_abs_delta_raw_logits": 0.0067774298950098455,
   "logits_min": -7.7265625,
   "logits_max": 3.326171875,
   "raw_logits_min": -7.7265625,
   "raw_logits_max": 3.326171875,
   "raw_logits_absmax": 7.7265625,
   "oracle_logits_min": -7.726421356201172,
   "oracle_logits_max": 3.365875244140625,
   "oracle_raw_logits_absmax": 7.726421356201172,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.8945889160968363,
   "seconds": 0.8945889160968363,
   "full_vocab_argmax_id": 3,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r16-position",
   "request_id": "r16",
   "question_id": "position",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 81,
   "nopts": 2,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    -2.486328125,
    1.8984375,
    -1.2880859375,
    -2.111328125,
    -1.7919921875,
    -2.763671875,
    -2.390625,
    -3.056640625,
    -3.255859375,
    -1.6142578125,
    -2.162109375,
    -3.345703125,
    -4.02734375,
    -2.07421875,
    -3.224609375,
    -4.0703125,
    -3.791015625,
    -4.4453125,
    -2.68359375,
    -2.44140625,
    -3.328125,
    -1.5654296875,
    -4.76171875,
    -6.203125,
    -3.072265625,
    -4.4375,
    -2.05078125,
    -4.48828125,
    -3.91015625,
    -5.796875,
    -4.33984375,
    -3.580078125,
    -5.74609375,
    -4.32421875,
    -2.6484375,
    -4.86328125,
    -4.97265625,
    -5.7890625,
    -4.0546875,
    -2.62890625,
    -6.6875,
    -4.03125,
    -4.375,
    -5.73828125,
    -3.583984375,
    -4.46484375,
    -4.03125,
    -4.19140625,
    -4.99609375,
    -4.1328125,
    -5.39453125,
    -4.5546875,
    -2.328125,
    -4.671875,
    -2.478515625,
    -3.357421875,
    -3.50390625,
    -4.45703125,
    -4.60546875,
    -5.23828125,
    -7.01171875,
    -5.97265625,
    -7.42578125,
    -6.26171875,
    -3.0625,
    -6.9140625,
    -7.00390625,
    -2.38671875,
    -7.3203125,
    -4.12890625,
    -7.44140625,
    -4.77734375,
    -2.291015625,
    -5.37109375,
    -3.49609375,
    -5.50390625,
    -3.765625,
    -6.21484375,
    -3.716796875,
    -5.06640625,
    -4.171875,
    -3.421875,
    -2.171875,
    -3.9609375,
    -2.974609375,
    -3.73828125,
    -2.19140625,
    -3.517578125,
    -2.81640625,
    -4.0390625,
    -2.994140625,
    -3.7578125,
    -3.712890625,
    -3.556640625,
    -4.27734375,
    -4.34375,
    -4.6640625,
    -3.330078125,
    -3.3046875,
    -3.78515625,
    -5.046875,
    -4.1953125,
    -3.322265625,
    -3.5859375,
    -4.04296875,
    -5.44140625,
    -3.484375,
    -4.953125,
    -3.619140625,
    -5.0703125,
    -4.08203125,
    -4.1484375,
    -4.51953125,
    -5.609375,
    -4.26171875,
    -3.466796875,
    -4.2421875,
    -5.30859375,
    -3.916015625,
    -2.890625,
    -3.048828125,
    -3.78515625,
    -2.953125,
    -3.53125,
    -2.2421875,
    -5.3125,
    -4.84765625,
    -2.76953125,
    -2.705078125,
    -4.8515625,
    -5.1796875,
    -2.724609375,
    -5.13671875,
    -5.60546875,
    -4.83203125,
    -3.955078125,
    -3.40234375,
    -4.40234375,
    -5.41796875,
    -2.537109375,
    -3.060546875,
    -3.73046875,
    -3.2265625,
    -3.052734375,
    -4.41796875,
    -4.7421875,
    -5.34375,
    -4.62890625,
    -4.57421875,
    -4.04296875,
    -4.3671875,
    -2.779296875,
    -0.61572265625,
    -3.1015625,
    -2.099609375,
    -1.0712890625,
    -1.4697265625,
    -1.96875,
    -3.185546875,
    -0.279052734375,
    -2.080078125,
    -2.109375,
    -1.2373046875,
    -1.06640625,
    -2.587890625,
    -2.046875,
    -2.462890625,
    -1.439453125,
    -3.19921875,
    -1.7958984375,
    -1.8671875,
    -2.16015625,
    -1.9833984375,
    -2.6875,
    -0.4150390625,
    -1.4248046875,
    -2.94140625,
    -3.072265625,
    -2.65625,
    -2.74609375,
    -2.8828125,
    -2.26953125,
    -2.232421875,
    -1.62890625,
    -2.109375,
    -1.302734375,
    -1.0986328125,
    -1.466796875,
    -1.5595703125,
    -1.6416015625,
    -0.10955810546875,
    -1.443359375,
    -2.447265625,
    -1.2578125,
    -1.216796875,
    -1.7861328125,
    -1.5927734375,
    -1.37890625,
    -1.2353515625,
    -0.98388671875,
    -0.9541015625,
    -1.47265625,
    -1.3974609375,
    -1.3603515625,
    -1.9912109375,
    -0.27734375,
    -2.28515625,
    -2.083984375,
    -0.26318359375,
    -1.78125,
    -0.351318359375,
    -0.826171875,
    -0.2088623046875,
    -0.96630859375,
    -2.158203125,
    -0.869140625,
    -1.39453125,
    -2.0703125,
    -1.4248046875,
    -1.4365234375,
    -1.212890625,
    -1.6640625,
    -1.2783203125,
    -1.1513671875,
    -0.07830810546875,
    0.342529296875,
    -0.400390625,
    1.04296875,
    1.0302734375,
    0.081298828125,
    -0.60693359375,
    -0.446044921875,
    0.03594970703125,
    -0.826171875,
    0.438720703125,
    1.267578125,
    1.4638671875,
    -0.52685546875,
    -0.27197265625,
    -0.91455078125,
    0.5615234375,
    0.71142578125,
    -0.67529296875,
    -0.03924560546875,
    -0.2147216796875,
    0.52587890625,
    0.444091796875,
    -0.3564453125,
    -0.931640625,
    1.71875,
    -0.2073974609375,
    -0.85693359375,
    -0.1929931640625,
    0.294921875,
    -0.578125,
    -6.421875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.014719272963702679,
    0.9849423170089722,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003383658768143505
   ],
   "p": [
    0.014724256470799446,
    0.9852757453918457
   ],
   "p_oracle": [
    0.015016984194517136,
    0.9849830269813538
   ],
   "abstain": 0.0003383658768143505,
   "abstain_oracle": 0.0003404226154088974,
   "abs_delta_abstain": 2.056738594546914e-06,
   "oracle_margin": 0.9699660539627075,
   "max_abs_delta_p": 0.0002927277237176895,
   "mean_abs_delta_p": 0.00029272306710481644,
   "max_abs_delta_raw_logits": 0.02702498435974121,
   "mean_abs_delta_raw_logits": 0.006352421303745359,
   "logits_min": -7.44140625,
   "logits_max": 1.8984375,
   "raw_logits_min": -7.44140625,
   "raw_logits_max": 1.8984375,
   "raw_logits_absmax": 7.44140625,
   "oracle_logits_min": -7.428675651550293,
   "oracle_logits_max": 1.8974967002868652,
   "oracle_raw_logits_absmax": 7.428675651550293,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.46000091696623713,
   "seconds": 0.46000091696623713,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r17-slot",
   "request_id": "r17",
   "question_id": "slot",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 217,
   "nopts": 40,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.87109375,
    -1.8134765625,
    -0.203125,
    -4.3359375,
    -2.033203125,
    -2.892578125,
    -4.45703125,
    -2.681640625,
    -5.171875,
    -2.978515625,
    -3.58203125,
    -3.60546875,
    -6.0546875,
    -5.1796875,
    -4.3203125,
    -3.51953125,
    -3.615234375,
    -5.84765625,
    -3.86328125,
    -4.7734375,
    -5.046875,
    -3.71484375,
    -5.6328125,
    -5.59375,
    -4.1484375,
    -2.25390625,
    -3.25390625,
    5.33203125,
    -4.26171875,
    -6.58984375,
    -3.62109375,
    -7.25390625,
    -4.10546875,
    -5.74609375,
    -4.6015625,
    -5.94921875,
    -7.2578125,
    -7.40234375,
    -7.125,
    -4.06640625,
    -6.56640625,
    -6.234375,
    -6.12890625,
    -6.8359375,
    -4.921875,
    -6.26171875,
    -4.70703125,
    -4.56640625,
    -5.54296875,
    -6.2890625,
    -4.8046875,
    -5.7421875,
    -5.11328125,
    -4.59765625,
    -4.65234375,
    -4.66015625,
    -4.6640625,
    -4.734375,
    -5.3515625,
    -6.609375,
    -8.0703125,
    -7.546875,
    -9.0,
    -6.61328125,
    -4.70703125,
    -8.6640625,
    -8.96875,
    -2.57421875,
    -9.9921875,
    -5.30859375,
    -8.6640625,
    -5.8828125,
    -2.8515625,
    -7.2109375,
    -3.705078125,
    -6.765625,
    -4.6328125,
    -7.78515625,
    -4.36328125,
    -6.03125,
    -3.59375,
    -4.921875,
    -3.646484375,
    -4.62890625,
    -4.21484375,
    -5.4453125,
    -3.30859375,
    -4.95703125,
    -3.697265625,
    -5.58203125,
    -4.98046875,
    -3.849609375,
    -4.0859375,
    -5.16015625,
    -5.578125,
    -5.6640625,
    -5.98046875,
    -3.5390625,
    -3.42578125,
    -5.8203125,
    -7.0,
    -5.0234375,
    -3.57421875,
    -5.74609375,
    -5.51953125,
    -7.27734375,
    -4.41796875,
    -6.0546875,
    -4.45703125,
    -7.4765625,
    -5.28515625,
    -5.7421875,
    -5.98046875,
    -6.52734375,
    -5.2109375,
    -2.845703125,
    -5.08984375,
    -7.0,
    -5.578125,
    -4.1640625,
    -4.30859375,
    -4.234375,
    -3.126953125,
    -3.29296875,
    -2.716796875,
    -6.22265625,
    -7.16015625,
    -3.240234375,
    -2.974609375,
    -6.0546875,
    -5.23828125,
    -2.6171875,
    -6.46875,
    -7.03515625,
    -5.46875,
    -5.6640625,
    -3.86328125,
    -5.26171875,
    -7.56640625,
    -3.513671875,
    -3.1328125,
    -4.33984375,
    -4.03125,
    -2.642578125,
    -4.1171875,
    -4.75390625,
    -6.92578125,
    -6.53515625,
    -6.15625,
    -5.0859375,
    -4.51953125,
    -3.283203125,
    -1.2998046875,
    -2.677734375,
    -1.7119140625,
    -2.01171875,
    -1.94921875,
    -1.2333984375,
    -3.115234375,
    0.327392578125,
    -1.328125,
    -1.6298828125,
    -1.1005859375,
    -1.7978515625,
    -2.45703125,
    -1.7646484375,
    -3.537109375,
    -1.7451171875,
    -5.19921875,
    -2.67578125,
    -3.5234375,
    -2.314453125,
    -1.99609375,
    -3.65234375,
    -1.5166015625,
    -3.353515625,
    -3.25390625,
    -3.408203125,
    -3.326171875,
    -4.28125,
    -4.19140625,
    -2.74609375,
    -2.515625,
    -3.0859375,
    -2.888671875,
    -2.611328125,
    -2.970703125,
    -1.822265625,
    -3.294921875,
    -2.658203125,
    -2.490234375,
    -2.265625,
    -3.486328125,
    -2.857421875,
    -1.8173828125,
    -1.701171875,
    -1.8505859375,
    -2.19921875,
    -2.16015625,
    -1.0712890625,
    -1.15625,
    -3.20703125,
    -1.609375,
    -1.791015625,
    -2.82421875,
    -1.560546875,
    -2.330078125,
    -3.060546875,
    0.189208984375,
    -2.525390625,
    -1.0009765625,
    -1.267578125,
    -1.0986328125,
    -1.9990234375,
    -4.06640625,
    -2.0078125,
    -1.328125,
    -2.4140625,
    -1.2333984375,
    -2.01953125,
    -1.7783203125,
    -2.248046875,
    -2.220703125,
    -0.96875,
    0.461669921875,
    -0.57958984375,
    -0.0936279296875,
    0.358642578125,
    0.892578125,
    -1.244140625,
    0.258056640625,
    0.261962890625,
    0.050048828125,
    -0.7080078125,
    0.155517578125,
    1.37109375,
    1.3359375,
    -0.234619140625,
    0.41015625,
    -1.00390625,
    0.28564453125,
    0.6328125,
    -0.19580078125,
    -0.00981903076171875,
    0.182373046875,
    -0.01959228515625,
    0.4248046875,
    -0.03143310546875,
    -1.28515625,
    2.16015625,
    -0.82958984375,
    0.6435546875,
    -0.475830078125,
    0.099365234375,
    0.1785888671875,
    -4.46484375
   ],
   "raw256_argmax": 27,
   "option_argmax": 27,
   "oracle_argmax": 27,
   "argmax_agrees": true,
   "p_full": [
    0.0010932465083897114,
    0.001154402270913124,
    0.005285074934363365,
    0.00010652315540937707,
    0.0009380109841004014,
    0.00041650887578725815,
    9.500822488917038e-05,
    0.0005083554424345493,
    4.8358859203290194e-05,
    0.00038403065991587937,
    0.00021714811737183481,
    0.00021239278430584818,
    2.100275560223963e-05,
    4.800321767106652e-05,
    0.00010810719686560333,
    0.00023035546473693103,
    0.00021044249297119677,
    2.553977356001269e-05,
    0.00016648101154714823,
    7.046063547022641e-05,
    5.4420299420598894e-05,
    0.00019154264009557664,
    3.1286912417272106e-05,
    3.2463074603583664e-05,
    0.00012716608762275428,
    0.0007614791393280029,
    0.0002960593847092241,
    0.9863044619560242,
    0.00011425999400671571,
    1.2668136150750797e-05,
    0.00020928066805936396,
    6.764901627320796e-06,
    0.00013243424473330379,
    2.8111600840929896e-05,
    8.288257959065959e-05,
    2.3203188902698457e-05,
    6.739986474713078e-06,
    5.879774562345119e-06,
    7.640982403245289e-06,
    0.0001374126732116565,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    9.430968202650547e-05
   ],
   "p": [
    0.0010933497687801719,
    0.001154511352069676,
    0.005285574123263359,
    0.00010653321805875748,
    0.0009380996343679726,
    0.00041654822416603565,
    9.501720342086628e-05,
    0.0005084034637548029,
    4.83634285046719e-05,
    0.0003840669523924589,
    0.0002171686355723068,
    0.0002124128513969481,
    2.10047401196789e-05,
    4.8007754230638966e-05,
    0.00010811741231009364,
    0.00023037723440211266,
    0.00021046237088739872,
    2.5542187358951196e-05,
    0.0001664967421675101,
    7.046729297144338e-05,
    5.4425439884653315e-05,
    0.00019156074267812073,
    3.128987009404227e-05,
    3.246614141971804e-05,
    0.0001271781075047329,
    0.000761551083996892,
    0.000296087353490293,
    0.9863976240158081,
    0.00011427079152781516,
    1.266933304577833e-05,
    0.0002093004441121593,
    6.765541002096143e-06,
    0.00013244675938040018,
    2.8114256565459073e-05,
    8.289040852105245e-05,
    2.320538078492973e-05,
    6.740623121004319e-06,
    5.8803302636079025e-06,
    7.641704542038497e-06,
    0.00013742565352004021
   ],
   "p_oracle": [
    0.0011019704397767782,
    0.0011741779744625092,
    0.005572291556745768,
    0.0001043995944201015,
    0.0009337730007246137,
    0.00043312847265042365,
    9.309892629971728e-05,
    0.0005304862861521542,
    4.843987335334532e-05,
    0.00039639012538827956,
    0.00021776325593236834,
    0.0002123292360920459,
    2.0675977793871425e-05,
    4.703338709077798e-05,
    0.00011001290840795264,
    0.0002325861423742026,
    0.0002133888629032299,
    2.58428535744315e-05,
    0.00017119513358920813,
    7.047851977404207e-05,
    5.554829476750456e-05,
    0.0001989987795241177,
    3.199079947080463e-05,
    3.3029155019903556e-05,
    0.00012720674567390233,
    0.0007189783500507474,
    0.0002877834776882082,
    0.9860422611236572,
    0.00011699549941113219,
    1.294876892643515e-05,
    0.0002151035296265036,
    6.854175808257423e-06,
    0.00014207363710738719,
    2.838669388438575e-05,
    8.500603871652856e-05,
    2.3704154955339618e-05,
    6.847472377558006e-06,
    5.990691533952486e-06,
    7.608487067045644e-06,
    0.00014321567141450942
   ],
   "abstain": 9.430968202650547e-05,
   "abstain_oracle": 9.498046711087227e-05,
   "abs_delta_abstain": 6.707850843667984e-07,
   "oracle_margin": 0.9804699420928955,
   "max_abs_delta_p": 0.0003553628921508789,
   "mean_abs_delta_p": 2.0800287495603696e-05,
   "max_abs_delta_raw_logits": 0.07665181159973145,
   "mean_abs_delta_raw_logits": 0.010398604877991602,
   "logits_min": -9.9921875,
   "logits_max": 5.33203125,
   "raw_logits_min": -9.9921875,
   "raw_logits_max": 5.33203125,
   "raw_logits_absmax": 9.9921875,
   "oracle_logits_min": -9.988116264343262,
   "oracle_logits_max": 5.315891265869141,
   "oracle_raw_logits_absmax": 9.988116264343262,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 1.3951326669193804,
   "seconds": 1.3951326669193804,
   "full_vocab_argmax_id": 27,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r18-slot",
   "request_id": "r18",
   "question_id": "slot",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 1449,
   "nopts": 255,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -5.0859375,
    -5.1015625,
    -7.3046875,
    -6.6328125,
    -7.515625,
    -8.1171875,
    -7.09765625,
    -8.4140625,
    -9.453125,
    -6.34375,
    -9.140625,
    -9.25,
    -7.39453125,
    -9.4296875,
    -9.078125,
    -8.0859375,
    -10.1484375,
    -9.3671875,
    -7.83203125,
    -10.1875,
    -9.34375,
    -7.8671875,
    -10.3203125,
    -10.109375,
    -11.1796875,
    -12.0,
    -10.3984375,
    -11.1328125,
    -10.1328125,
    -10.3515625,
    -9.515625,
    -13.2890625,
    -11.375,
    -10.3046875,
    -9.765625,
    -11.5859375,
    -11.3515625,
    -9.1875,
    -9.578125,
    -7.71484375,
    -11.09375,
    -11.0234375,
    -12.375,
    -10.203125,
    -10.3515625,
    -11.9140625,
    -9.953125,
    -9.8828125,
    -10.609375,
    -10.6640625,
    -13.078125,
    -11.5,
    -12.3984375,
    -11.1015625,
    -10.9140625,
    -9.2890625,
    -9.40625,
    -8.21875,
    -9.203125,
    -7.03125,
    -14.796875,
    -13.7109375,
    -14.921875,
    -11.0390625,
    -11.0546875,
    -18.140625,
    -16.796875,
    -8.9765625,
    -17.21875,
    -10.9375,
    -16.125,
    -10.0703125,
    -7.26171875,
    -12.0,
    -9.1328125,
    -15.0,
    -10.046875,
    -14.3515625,
    -8.3984375,
    -12.046875,
    -8.6875,
    -10.78125,
    -7.0078125,
    -12.5546875,
    -8.671875,
    -9.9765625,
    -8.578125,
    -9.90625,
    -9.390625,
    -14.5078125,
    -11.640625,
    -13.1015625,
    -10.140625,
    -10.0703125,
    -9.6953125,
    -11.5,
    -9.71875,
    -7.0546875,
    -11.859375,
    -9.8671875,
    -16.03125,
    -8.6484375,
    -8.78125,
    -10.765625,
    -8.7421875,
    -15.8203125,
    -11.0078125,
    -12.3671875,
    -9.828125,
    -14.2890625,
    -7.98046875,
    -14.2734375,
    -14.390625,
    -15.5625,
    -10.296875,
    -6.65234375,
    -10.1171875,
    -15.21875,
    -12.875,
    -10.140625,
    -8.15625,
    -7.0,
    -7.98046875,
    -9.375,
    -7.27734375,
    -14.9296875,
    -15.421875,
    -9.609375,
    -8.84375,
    -16.4375,
    -12.0546875,
    -11.671875,
    -16.171875,
    -16.0,
    -8.1953125,
    -15.2421875,
    -10.65625,
    -10.2734375,
    -16.203125,
    -12.625,
    -9.7734375,
    -11.609375,
    -10.015625,
    -9.9375,
    -9.1796875,
    -10.4140625,
    -16.203125,
    -14.390625,
    -11.328125,
    -10.3828125,
    -11.6328125,
    -10.6484375,
    -10.796875,
    -10.1640625,
    -5.9609375,
    -5.78125,
    -4.89453125,
    -5.34765625,
    -7.4609375,
    -9.0,
    -6.65234375,
    -6.421875,
    -6.29296875,
    -6.29296875,
    -9.5,
    -3.615234375,
    -7.05078125,
    -7.1953125,
    -10.8515625,
    -8.1796875,
    -7.55859375,
    -7.5234375,
    -4.46484375,
    -9.234375,
    -7.38671875,
    -7.5,
    -10.9296875,
    -9.671875,
    -11.3203125,
    -10.0859375,
    -10.921875,
    -12.8515625,
    -10.0625,
    -10.734375,
    -11.453125,
    -9.7578125,
    -9.9296875,
    -10.0859375,
    -11.03125,
    -11.6328125,
    -11.34375,
    -9.8203125,
    -10.6875,
    -9.4609375,
    -11.3125,
    -10.3515625,
    -11.3515625,
    -9.3203125,
    -9.7734375,
    -10.1328125,
    -9.6015625,
    -10.5234375,
    -8.3046875,
    -11.046875,
    -9.7109375,
    -8.7578125,
    -10.171875,
    -8.796875,
    -0.72119140625,
    -10.71875,
    -8.859375,
    -9.7890625,
    -9.3671875,
    -9.0703125,
    -9.3359375,
    -9.046875,
    -10.265625,
    -8.5078125,
    -8.984375,
    -5.8828125,
    -7.05859375,
    -0.03045654296875,
    -1.78125,
    -1.072265625,
    1.5087890625,
    -0.83447265625,
    0.1378173828125,
    -0.45263671875,
    -0.7734375,
    0.2568359375,
    0.8896484375,
    1.4951171875,
    -1.9375,
    -0.1380615234375,
    1.181640625,
    0.74462890625,
    0.370361328125,
    -0.57763671875,
    0.86962890625,
    0.382080078125,
    -0.036529541015625,
    -1.2607421875,
    -1.34765625,
    1.1630859375,
    -0.26513671875,
    1.732421875,
    -1.1767578125,
    0.1639404296875,
    -1.3857421875,
    -0.2169189453125,
    -1.4521484375,
    1.1376953125,
    -0.2081298828125,
    1.818359375,
    -1.2392578125,
    -4.66796875
   ],
   "raw256_argmax": 253,
   "option_argmax": 253,
   "oracle_argmax": 253,
   "argmax_agrees": true,
   "p_full": [
    0.00015897859702818096,
    0.00015664915554225445,
    1.9544810129445978e-05,
    3.687126445583999e-05,
    1.6013573258533143e-05,
    9.071506610780489e-06,
    2.3766882804920897e-05,
    6.852954811620293e-06,
    2.5678637030068785e-06,
    4.844893919653259e-05,
    3.449722271398059e-06,
    3.111070782324532e-06,
    1.795437310647685e-05,
    2.625354227347998e-06,
    3.65954088010767e-06,
    9.343307283415925e-06,
    1.3313723457031301e-06,
    2.7850330752698937e-06,
    1.1876082680828404e-05,
    1.283136839447252e-06,
    2.8473853035393404e-06,
    1.1488135896797758e-05,
    1.1318342103550094e-06,
    1.3814209296469926e-06,
    5.025732434660313e-07,
    2.3154851191975467e-07,
    1.0513075494600344e-06,
    5.253287440609711e-07,
    1.3511704537449987e-06,
    1.0989086831614259e-06,
    2.4206365196732804e-06,
    6.851195877288774e-08,
    4.1789422766669304e-07,
    1.1486650919323438e-06,
    1.9114363567496184e-06,
    3.4239161550431163e-07,
    4.2725019966383115e-07,
    3.300291837149416e-06,
    2.2818501292931614e-06,
    1.3266389942145906e-05,
    5.450767730508232e-07,
    5.82512768687593e-07,
    1.624753167561721e-07,
    1.2643356512853643e-06,
    1.0989086831614259e-06,
    2.511310128738842e-07,
    1.6011518937375513e-06,
    1.7111191255025915e-06,
    8.613637305643351e-07,
    8.179927135643084e-07,
    8.361989500826894e-08,
    3.7134833519303356e-07,
    1.589172740068534e-07,
    5.410686867435288e-07,
    6.459214318965678e-07,
    2.9983573313074885e-06,
    2.684131686692126e-06,
    8.241579962486867e-06,
    3.2519340038561495e-06,
    2.5305627787020057e-05,
    1.6487009091292748e-08,
    4.5991711772330746e-08,
    1.4650654023284915e-08,
    5.739774451285484e-07,
    5.655671770909976e-07,
    7.002780022524746e-10,
    2.4922042207720096e-09,
    4.028056537208613e-06,
    1.673004845770265e-09,
    6.317769702945952e-07,
    4.701536049367405e-09,
    1.4333510307551478e-06,
    2.0354518710519187e-05,
    2.3154851191975467e-07,
    3.475277253528475e-06,
    1.3608303817136402e-08,
    1.4654414144388284e-06,
    2.5109880397167217e-08,
    6.9548609644698445e-06,
    2.2151860434860282e-07,
    5.292881724017207e-06,
    7.322677220145124e-07,
    2.5872179321595468e-05,
    1.3710869950500637e-07,
    5.371589850255987e-06,
    1.56608962242899e-06,
    5.869032065675128e-06,
    1.6736488532842486e-06,
    2.7240459985478083e-06,
    2.1663996818688247e-08,
    3.251516886848549e-07,
    8.178877664022366e-08,
    1.341234792562318e-06,
    1.4333510307551478e-06,
    2.0427139588719e-06,
    3.7134833519303356e-07,
    1.9979825083282776e-06,
    2.475148176017683e-05,
    2.6444629952493415e-07,
    1.7365642861477681e-06,
    5.1369273279533445e-09,
    5.4918505156820174e-06,
    4.844272552873008e-06,
    7.431568178617454e-07,
    5.026377493777545e-06,
    6.269703867189946e-09,
    5.91174966757535e-07,
    1.6367890509627614e-07,
    1.8018448599832482e-06,
    2.66371085189121e-08,
    1.0322204616386443e-05,
    2.7033214777816283e-08,
    2.4200152992648327e-08,
    7.998757212135388e-09,
    1.1571740969884559e-06,
    3.619716517277993e-05,
    1.3712628970097285e-06,
    1.1067661453978417e-08,
    1.0130888483672607e-07,
    1.341234792562318e-06,
    8.742847057874314e-06,
    2.6063835321110673e-05,
    1.0322204616386443e-05,
    2.764553528322722e-06,
    2.0056273569935e-05,
    1.4542924198224227e-08,
    9.135198375531672e-09,
    2.2154702037369134e-06,
    4.566523784887977e-06,
    3.4996741060666636e-09,
    2.198896993377275e-07,
    3.156928869429976e-07,
    4.497880734533055e-09,
    5.290839766303179e-09,
    8.426095519098453e-06,
    1.0825289997740128e-08,
    8.240522788582894e-07,
    1.1830815083158086e-06,
    4.367035622010462e-09,
    1.2829723061713594e-07,
    1.8973807982547441e-06,
    3.3489388329144276e-07,
    1.5093504543983727e-06,
    1.6249616692221025e-06,
    3.3247395094804233e-06,
    1.035903096635593e-06,
    4.367035622010462e-09,
    2.4200152992648327e-08,
    4.3681563965947134e-07,
    1.066940967575647e-06,
    3.2756034329395334e-07,
    8.301566367663327e-07,
    7.215381288006029e-07,
    1.3118642527842894e-06,
    6.955753633519635e-05,
    8.242636977229267e-05,
    0.00019048851390834898,
    0.00012415401579346508,
    1.6862633856362663e-05,
    3.9398496483045164e-06,
    3.619716517277993e-05,
    4.5001939724897966e-05,
    5.082981806481257e-05,
    5.082981806481257e-05,
    2.456632273606374e-06,
    0.0006378780817613006,
    2.4843000574037433e-05,
    2.167233469663188e-05,
    6.852075671304192e-07,
    8.551394785172306e-06,
    1.5376548617496155e-05,
    1.5895819160505198e-05,
    0.0002858646912500262,
    3.15733382194594e-06,
    1.808737579267472e-05,
    1.625170261831954e-05,
    6.364570026562433e-07,
    2.08844699045585e-06,
    4.4005147969983227e-07,
    1.412348751728132e-06,
    6.411717663468153e-07,
    1.0357702961982795e-07,
    1.4439689266509959e-06,
    7.654233513676445e-07,
    3.88162249009838e-07,
    1.9255958250141703e-06,
    1.6369990589737426e-06,
    1.412348751728132e-06,
    5.782293897027557e-07,
    3.2756034329395334e-07,
    4.3041515596087265e-07,
    1.8151926042264677e-06,
    8.000802154128905e-07,
    2.5489814561296953e-06,
    4.4331127924124303e-07,
    1.0989086831614259e-06,
    4.2725019966383115e-07,
    2.9111336061760085e-06,
    1.8973807982547441e-06,
    1.3511704537449987e-06,
    2.2318818082567304e-06,
    9.34210845571215e-07,
    7.5989241850038525e-06,
    5.697568212781334e-07,
    2.012782942983904e-06,
    4.952727977070026e-06,
    1.3022176972299349e-06,
    4.773291493620491e-06,
    0.009819820523262024,
    7.768055070300761e-07,
    4.4996127144258935e-06,
    1.869579477897787e-06,
    2.7850330752698937e-06,
    3.6866499613097403e-06,
    2.8684780772891827e-06,
    3.7691881971113617e-06,
    1.1918455129489303e-06,
    6.272117843764136e-06,
    3.998437023255974e-06,
    7.488540722988546e-05,
    2.466032310621813e-05,
    0.018858090043067932,
    0.0036073101218789816,
    0.007047985680401325,
    0.08072690665721893,
    0.008823208510875702,
    0.02210736647248268,
    0.012655693106353283,
    0.009346908889710903,
    0.024738186970353127,
    0.04497775062918663,
    0.07969095557928085,
    0.0031122700311243534,
    0.017035294324159622,
    0.05926470085978508,
    0.039219241589307785,
    0.027538800612092018,
    0.011246077716350555,
    0.04413510486483574,
    0.027845365926623344,
    0.018750209361314774,
    0.005898437462747097,
    0.005433478392660618,
    0.05823491886258125,
    0.015108221210539341,
    0.09971723705530167,
    0.00638548843562603,
    0.02265973389148712,
    0.0052414583042263985,
    0.015812337398529053,
    0.004922741558402777,
    0.05685468390583992,
    0.01594417355954647,
    0.10815051198005676,
    0.006019379012286663,
    0.00023595143284182996
   ],
   "p": [
    0.00015901612641755491,
    0.00015668613195884973,
    1.954942308657337e-05,
    3.6879966501146555e-05,
    1.601735311851371e-05,
    9.073647561308462e-06,
    2.3772492568241432e-05,
    6.854572347947396e-06,
    2.56846988122561e-06,
    4.846037336392328e-05,
    3.4505364965298213e-06,
    3.1118051992962137e-06,
    1.7958611351787113e-05,
    2.6259738206135808e-06,
    3.660404672700679e-06,
    9.345512808067724e-06,
    1.3316865761225927e-06,
    2.7856904125656e-06,
    1.1878885743499268e-05,
    1.2834397011829424e-06,
    2.8480574201239506e-06,
    1.1490847100503743e-05,
    1.1321013744236552e-06,
    1.3817469834975782e-06,
    5.026918756811938e-07,
    2.3160316686698934e-07,
    1.0515557278267806e-06,
    5.254527195575065e-07,
    1.3514893453248078e-06,
    1.0991680028382689e-06,
    2.4212079097196693e-06,
    6.852813072555364e-08,
    4.1799285099841654e-07,
    1.1489362350403098e-06,
    1.911887466121698e-06,
    3.4247241842422227e-07,
    4.2735103988889023e-07,
    3.3010708193614846e-06,
    2.2823887775302865e-06,
    1.326952133240411e-05,
    5.452054097077053e-07,
    5.826502729178173e-07,
    1.6251367185304844e-07,
    1.2646340792343835e-06,
    1.0991680028382689e-06,
    2.51190300559756e-07,
    1.601529788786138e-06,
    1.7115230548370164e-06,
    8.615670594736002e-07,
    8.181858106581785e-07,
    8.363963388546836e-08,
    3.7143598774491693e-07,
    1.5895477645244682e-07,
    5.41196413905709e-07,
    6.460738859459525e-07,
    2.9990651455591433e-06,
    2.684765149751911e-06,
    8.24352537165396e-06,
    3.252701617384446e-06,
    2.5311601348221302e-05,
    1.6490901089127874e-08,
    4.600256886533316e-08,
    1.4654111701872807e-08,
    5.741129029956937e-07,
    5.657006454384828e-07,
    7.004433144608413e-10,
    2.492792416930456e-09,
    4.029007413919317e-06,
    1.6733997521001243e-09,
    6.319260705822671e-07,
    4.7026458283028205e-09,
    1.4336893627842073e-06,
    2.0359322661533952e-05,
    2.3160316686698934e-07,
    3.476097617749474e-06,
    1.3611515470302038e-08,
    1.4657872497991775e-06,
    2.5115806323583456e-08,
    6.956502602406545e-06,
    2.2157088608310005e-07,
    5.294130914990092e-06,
    7.324405828512681e-07,
    2.587828566902317e-05,
    1.3714105762119289e-07,
    5.372857685870258e-06,
    1.566459218338423e-06,
    5.870417226105928e-06,
    1.6740439150453312e-06,
    2.7246890113019617e-06,
    2.166911095002888e-08,
    3.25228427300317e-07,
    8.180808208635426e-08,
    1.3415514104053727e-06,
    1.4336893627842073e-06,
    2.0431959910638398e-06,
    3.7143598774491693e-07,
    1.998454081331147e-06,
    2.475732435414102e-05,
    2.645087135988433e-07,
    1.7369741271977546e-06,
    5.138139691496235e-09,
    5.493146545632044e-06,
    4.8454157877131365e-06,
    7.433322366523498e-07,
    5.027563929616008e-06,
    6.271183572437167e-09,
    5.913145173508383e-07,
    1.6371754441024677e-07,
    1.8022701624431647e-06,
    2.6643395045766738e-08,
    1.0324641152692493e-05,
    2.703959545158341e-08,
    2.420586575624384e-08,
    8.00064547945567e-09,
    1.1574471727726632e-06,
    3.6205707147018984e-05,
    1.371586563436722e-06,
    1.1070273586710755e-08,
    1.0133279459978439e-07,
    1.3415514104053727e-06,
    8.744910701352637e-06,
    2.6069987143273465e-05,
    1.0324641152692493e-05,
    2.765206090771244e-06,
    2.0061006580363028e-05,
    1.4546357007816368e-08,
    9.137354872734704e-09,
    2.215993163190433e-06,
    4.567601536109578e-06,
    3.5005001119969847e-09,
    2.1994159737914742e-07,
    3.157674086651241e-07,
    4.498942551833807e-09,
    5.2920885451612776e-09,
    8.42808458401123e-06,
    1.0827845287053606e-08,
    8.24246797037631e-07,
    1.183360723189253e-06,
    4.368066353066524e-09,
    1.2832751394853403e-07,
    1.8978286107085296e-06,
    3.349729240653687e-07,
    1.5097067489477922e-06,
    1.6253452486125752e-06,
    3.325524176034378e-06,
    1.0361476370235323e-06,
    4.368066353066524e-09,
    2.420586575624384e-08,
    4.3691875362128485e-07,
    1.0671927839212003e-06,
    3.2763765034360404e-07,
    8.303525760311459e-07,
    7.217084316835098e-07,
    1.312173935730243e-06,
    6.957395089557394e-05,
    8.2445825682953e-05,
    0.00019053347932640463,
    0.000124183323350735,
    1.686661380517762e-05,
    3.940779606637079e-06,
    3.6205707147018984e-05,
    4.5012562623014674e-05,
    5.084181611891836e-05,
    5.084181611891836e-05,
    2.457212076478754e-06,
    0.0006380286649800837,
    2.4848864995874465e-05,
    2.167744969483465e-05,
    6.853692866570782e-07,
    8.55341295391554e-06,
    1.538017750135623e-05,
    1.5899571735644713e-05,
    0.00028593215392902493,
    3.158079152854043e-06,
    1.8091644960804842e-05,
    1.625553886697162e-05,
    6.366072398122924e-07,
    2.088939936584211e-06,
    4.4015533262609097e-07,
    1.4126820815363317e-06,
    6.413230835278227e-07,
    1.0360147939536546e-07,
    1.4443097597904853e-06,
    7.656039997527841e-07,
    3.882538806010416e-07,
    1.9260503449913813e-06,
    1.6373854805351584e-06,
    1.4126820815363317e-06,
    5.783658707514405e-07,
    3.2763765034360404e-07,
    4.305167351503769e-07,
    1.8156210899178404e-06,
    8.002690492503461e-07,
    2.549583086874918e-06,
    4.434159279753658e-07,
    1.0991680028382689e-06,
    4.2735103988889023e-07,
    2.911820729423198e-06,
    1.8978286107085296e-06,
    1.3514893453248078e-06,
    2.2324086330627324e-06,
    9.344313411929761e-07,
    7.600717708555749e-06,
    5.698913128071581e-07,
    2.013257926591905e-06,
    4.953897132509155e-06,
    1.3025251064391341e-06,
    4.774418357555987e-06,
    0.009822138585150242,
    7.769888838993211e-07,
    4.500674549490213e-06,
    1.8700208102018223e-06,
    2.7856904125656e-06,
    3.6875201203656616e-06,
    2.8691551960946526e-06,
    3.770077910303371e-06,
    1.1921267741854535e-06,
    6.273598046391271e-06,
    3.9993806240090635e-06,
    7.490308053093031e-05,
    2.4666143872309476e-05,
    0.018862541764974594,
    0.003608161583542824,
    0.0070496490225195885,
    0.08074595779180527,
    0.008825290948152542,
    0.022112583741545677,
    0.012658679857850075,
    0.009349115192890167,
    0.024744026362895966,
    0.04498836770653725,
    0.07970976829528809,
    0.0031130046118050814,
    0.017039315775036812,
    0.059278689324855804,
    0.03922849893569946,
    0.027545301243662834,
    0.01124873198568821,
    0.04414552077651024,
    0.02785193920135498,
    0.018754635006189346,
    0.005899829789996147,
    0.005434760823845863,
    0.05824866518378258,
    0.015111787244677544,
    0.09974077343940735,
    0.006386995781213045,
    0.022665081545710564,
    0.005242695566266775,
    0.01581607013940811,
    0.0049239033833146095,
    0.05686810240149498,
    0.015947936102747917,
    0.10817603766918182,
    0.006020799744874239
   ],
   "p_oracle": [
    0.0001469101116526872,
    0.0001461399078834802,
    1.8727199858403765e-05,
    3.445339825702831e-05,
    1.491959483246319e-05,
    8.525082193955313e-06,
    2.203234726039227e-05,
    6.327398750727298e-06,
    2.4345908968825825e-06,
    4.6558663598261774e-05,
    3.151977125526173e-06,
    2.988307187479222e-06,
    1.7149484847323038e-05,
    2.3426230200129794e-06,
    3.3076789804908913e-06,
    8.558708032069262e-06,
    1.175259626506886e-06,
    2.548649490563548e-06,
    1.0875735824811272e-05,
    1.174447334051365e-06,
    2.7018331820727326e-06,
    1.073516978067346e-05,
    1.0747947953859693e-06,
    1.2419112636052887e-06,
    4.439075667050929e-07,
    2.042345670361101e-07,
    9.404049592376396e-07,
    4.902366299575078e-07,
    1.2479507631724118e-06,
    9.923292054736521e-07,
    2.2280037228483707e-06,
    5.986074569364064e-08,
    3.658147988971905e-07,
    1.0361229669797467e-06,
    1.7353581824863795e-06,
    3.016077982920251e-07,
    3.907217376308836e-07,
    2.925652097474085e-06,
    1.966859144886257e-06,
    1.2392310054565314e-05,
    5.118952799421095e-07,
    5.359339070309943e-07,
    1.4427850203446724e-07,
    1.123934225688572e-06,
    1.0238458116873517e-06,
    2.1720221354826208e-07,
    1.4275401554186828e-06,
    1.5700845779065276e-06,
    8.077286679508688e-07,
    7.595568831675337e-07,
    7.496300469256312e-08,
    3.4219951317027153e-07,
    1.4025690120433865e-07,
    5.061549472884508e-07,
    5.872788619853964e-07,
    2.820014060489484e-06,
    2.5274971449107397e-06,
    7.469083357136697e-06,
    2.887191612899187e-06,
    2.2984444512985647e-05,
    1.4499901723752373e-08,
    4.224759564408487e-08,
    1.3127833931037003e-08,
    5.267225446914381e-07,
    5.032041485719674e-07,
    5.929003399351984e-10,
    2.150327471639457e-09,
    3.6878504943160806e-06,
    1.5137613385007853e-09,
    5.771068458670925e-07,
    4.05874622799729e-09,
    1.283262236029259e-06,
    1.8895223547588103e-05,
    2.1562215124504291e-07,
    3.2647783427819377e-06,
    1.216329259534632e-08,
    1.3845196917827707e-06,
    2.249413455501781e-08,
    6.356727226375369e-06,
    2.0436063152828865e-07,
    4.80034123029327e-06,
    6.884560548314766e-07,
    2.4657274479977787e-05,
    1.261227424720346e-07,
    5.058232545707142e-06,
    1.4855725112283835e-06,
    5.549032721319236e-06,
    1.5243693951561e-06,
    2.5501326490484644e-06,
    1.9503026749134733e-08,
    2.894741442105442e-07,
    7.430752901882443e-08,
    1.2425402928784024e-06,
    1.359587031402043e-06,
    1.943861207109876e-06,
    3.3811633670666197e-07,
    1.8900060467785806e-06,
    2.3787959435139783e-05,
    2.389311362094304e-07,
    1.6118586927404976e-06,
    4.5472798859691466e-09,
    5.195611720409943e-06,
    4.406738753459649e-06,
    6.553700018230302e-07,
    4.563604306895286e-06,
    5.59060486793328e-09,
    5.400765417107323e-07,
    1.514987673090218e-07,
    1.6380289480366628e-06,
    2.4197854386898143e-08,
    9.32275997911347e-06,
    2.393656828303392e-08,
    2.1264302318968475e-08,
    6.9980532479974045e-09,
    1.081376012734836e-06,
    3.306658618384972e-05,
    1.222553464685916e-06,
    9.822396229708374e-09,
    9.009193746578603e-08,
    1.2590995766004198e-06,
    7.929240382509306e-06,
    2.503086216165684e-05,
    9.561872502672486e-06,
    2.482203854015097e-06,
    1.9406297724344768e-05,
    1.3104779483796847e-08,
    8.068131940319745e-09,
    2.05597734748153e-06,
    4.0540408008382656e-06,
    3.169926543478141e-09,
    1.9746653379115742e-07,
    2.882631235934241e-07,
    3.9420209319018795e-09,
    4.785592810918615e-09,
    8.075558071141131e-06,
    9.4671372963262e-09,
    7.656602178940375e-07,
    1.0820309626069502e-06,
    3.729021536003074e-09,
    1.2206049859742052e-07,
    1.735182763695775e-06,
    3.0206345513761335e-07,
    1.4078510730541893e-06,
    1.526253186057147e-06,
    3.32214199261216e-06,
    9.722509730636375e-07,
    3.903593004395134e-09,
    2.2041644953674222e-08,
    4.0664875200491224e-07,
    9.705481716082431e-07,
    2.8843609811701754e-07,
    7.440521017088031e-07,
    6.660162625848898e-07,
    1.2110650686736335e-06,
    6.550806574523449e-05,
    7.979811925906688e-05,
    0.0001818382297642529,
    0.00011510254262248054,
    1.574801717652008e-05,
    3.7079730645928066e-06,
    3.386830576346256e-05,
    4.226343662594445e-05,
    4.857313251704909e-05,
    4.7070567234186456e-05,
    2.2569599877897417e-06,
    0.0006120369071140885,
    2.3661097657168284e-05,
    2.0005791157018393e-05,
    6.244418386813777e-07,
    7.96031963545829e-06,
    1.4662005014542956e-05,
    1.4956691302359104e-05,
    0.0002772198058664799,
    2.891156555051566e-06,
    1.6640678950352594e-05,
    1.5442494259332307e-05,
    5.737019250773301e-07,
    1.9168603557773167e-06,
    3.9845309629527037e-07,
    1.2928065871165018e-06,
    5.787465511275514e-07,
    9.430754488448656e-08,
    1.3109068959238357e-06,
    6.849967917332833e-07,
    3.59871819455293e-07,
    1.7630660522627295e-06,
    1.5343573522841325e-06,
    1.2713866226476966e-06,
    5.288820261739602e-07,
    2.9492520070562023e-07,
    3.9059952428033284e-07,
    1.6423742863480584e-06,
    7.348767780968046e-07,
    2.3556704036309384e-06,
    4.0035379811342864e-07,
    9.642487839300884e-07,
    3.915128559128789e-07,
    2.6290808818885125e-06,
    1.7516496200187248e-06,
    1.2177035841887118e-06,
    2.065924718408496e-06,
    8.670426154822053e-07,
    6.833991392340977e-06,
    4.97766279750067e-07,
    1.818960640775913e-06,
    4.63209062218084e-06,
    1.1824243983937777e-06,
    4.360892944532679e-06,
    0.009755837731063366,
    7.05798640865396e-07,
    4.1220273487851955e-06,
    1.7139896044682246e-06,
    2.5792653559619794e-06,
    3.3125340905826306e-06,
    2.6990644528268604e-06,
    3.3724088552844478e-06,
    1.0858207133424003e-06,
    5.779492767032934e-06,
    3.6232527236279566e-06,
    6.950746319489554e-05,
    2.359280188102275e-05,
    0.018962303176522255,
    0.0035667235497385263,
    0.006895596627146006,
    0.08383864909410477,
    0.008625222370028496,
    0.022189911454916,
    0.012730501592159271,
    0.009463458321988583,
    0.024959232658147812,
    0.0444105863571167,
    0.08021362125873566,
    0.0028592399321496487,
    0.017242614179849625,
    0.060589153319597244,
    0.039358872920274734,
    0.02689761482179165,
    0.011007904075086117,
    0.044373709708452225,
    0.02796688862144947,
    0.0190079715102911,
    0.005807948764413595,
    0.005553585942834616,
    0.05795270577073097,
    0.014917745254933834,
    0.0977199524641037,
    0.006410952657461166,
    0.022017937153577805,
    0.005312574561685324,
    0.015614868141710758,
    0.004747352562844753,
    0.057182542979717255,
    0.01578008383512497,
    0.107459157705307,
    0.005929228384047747
   ],
   "abstain": 0.00023595143284182996,
   "abstain_oracle": 0.00022339480347000062,
   "abs_delta_abstain": 1.2556629371829331e-05,
   "oracle_margin": 0.009739205241203308,
   "max_abs_delta_p": 0.0030926913022994995,
   "mean_abs_delta_p": 5.445285578352905e-05,
   "max_abs_delta_raw_logits": 0.17230606079101562,
   "mean_abs_delta_raw_logits": 0.07908498687902465,
   "logits_min": -18.140625,
   "logits_max": 1.818359375,
   "raw_logits_min": -18.140625,
   "raw_logits_max": 1.818359375,
   "raw_logits_absmax": 18.140625,
   "oracle_logits_min": -18.312931060791016,
   "oracle_logits_max": 1.8154594898223877,
   "oracle_raw_logits_absmax": 18.312931060791016,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 8.328881041961722,
   "seconds": 8.328881041961722,
   "full_vocab_argmax_id": 253,
   "full_vocab_argmax_is_label": true
  }
 ],
 "sessions": [
  {
   "schema": "coreai-slot-readout-session/1",
   "session": "int8lin-00-kit-a1",
   "mode": "int8lin",
   "indices": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    12,
    13,
    14,
    15,
    16,
    17,
    18,
    19,
    20,
    21,
    22,
    23,
    24,
    25,
    26,
    27,
    28,
    29
   ],
   "pid": 14838,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_int8lin.h16c.aimodelc",
   "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
   "started_at": "2026-09-22T19:27:40+00:00",
   "phase": "complete",
   "runtime_calls": 2382,
   "planned_runtime_calls": 2382,
   "result": "PASS",
   "load_wall_seconds_contended": 1.311447334010154,
   "active_row": "r01-route",
   "active_step": 112,
   "reset_check": {
    "row": "r01-route",
    "identical": true,
    "max_abs_diff": 0.0,
    "first_raw_logits": [
     0.923828125,
     -4.08984375,
     -1.787109375,
     -3.15234375,
     -3.03515625,
     -3.1484375,
     -3.1015625,
     -3.046875,
     -4.265625,
     -2.974609375,
     -3.037109375,
     -2.724609375,
     -4.38671875,
     -3.40234375,
     -3.67578125,
     -3.583984375,
     -3.9765625,
     -4.8984375,
     -3.580078125,
     -3.421875,
     -3.69140625,
     -2.41015625,
     -5.51171875,
     -5.21875,
     -3.9375,
     -4.7890625,
     -3.427734375,
     -4.73828125,
     -4.14453125,
     -5.67578125,
     -4.3125,
     -4.5390625,
     -5.82421875,
     -3.943359375,
     -3.287109375,
     -4.94140625,
     -5.75,
     -6.21484375,
     -4.6875,
     -3.01953125,
     -5.88671875,
     -4.8046875,
     -5.15234375,
     -5.96484375,
     -3.826171875,
     -4.96484375,
     -4.35546875,
     -4.94921875,
     -5.39453125,
     -4.703125,
     -5.26953125,
     -4.6328125,
     -3.884765625,
     -5.47265625,
     -3.40625,
     -3.783203125,
     -3.955078125,
     -4.4453125,
     -4.58203125,
     -5.77734375,
     -7.453125,
     -7.21484375,
     -8.140625,
     -5.8203125,
     -3.728515625,
     -7.51171875,
     -8.03125,
     -2.712890625,
     -7.87109375,
     -4.35546875,
     -8.0078125,
     -5.390625,
     -2.765625,
     -5.41796875,
     -3.7734375,
     -6.3828125,
     -4.125,
     -7.0390625,
     -4.0390625,
     -6.0703125,
     -4.66015625,
     -4.265625,
     -3.19140625,
     -4.75,
     -3.564453125,
     -4.89453125,
     -2.65625,
     -4.12109375,
     -3.328125,
     -4.48828125,
     -4.02734375,
     -4.14453125,
     -4.31640625,
     -3.615234375,
     -4.78515625,
     -4.1171875,
     -5.1015625,
     -3.16015625,
     -2.66796875,
     -4.76171875,
     -6.18359375,
     -4.96875,
     -3.4140625,
     -4.359375,
     -4.7890625,
     -5.94921875,
     -3.48046875,
     -5.23828125,
     -4.140625,
     -6.0234375,
     -4.1796875,
     -4.98046875,
     -5.04296875,
     -5.79296875,
     -4.703125,
     -3.51171875,
     -4.7890625,
     -6.01953125,
     -4.265625,
     -3.701171875,
     -3.140625,
     -4.2890625,
     -2.89453125,
     -3.689453125,
     -2.212890625,
     -5.1171875,
     -5.84375,
     -2.671875,
     -2.82421875,
     -5.80078125,
     -5.6875,
     -3.255859375,
     -5.8203125,
     -6.65625,
     -5.3671875,
     -4.9609375,
     -3.77734375,
     -4.33984375,
     -6.3671875,
     -2.70703125,
     -3.63671875,
     -4.328125,
     -3.044921875,
     -2.8046875,
     -4.25390625,
     -4.93359375,
     -5.8984375,
     -5.80078125,
     -5.22265625,
     -4.51953125,
     -4.5546875,
     -2.87109375,
     -1.0556640625,
     -3.166015625,
     -1.939453125,
     -1.4638671875,
     -1.576171875,
     -1.51953125,
     -2.634765625,
     -0.87158203125,
     -1.861328125,
     -1.751953125,
     -1.4228515625,
     -1.3427734375,
     -2.6953125,
     -1.84765625,
     -2.994140625,
     -2.28515625,
     -3.892578125,
     -2.1171875,
     -2.21875,
     -2.548828125,
     -1.87890625,
     -2.92578125,
     -0.955078125,
     -1.955078125,
     -3.685546875,
     -3.427734375,
     -3.251953125,
     -2.583984375,
     -3.9296875,
     -2.513671875,
     -3.091796875,
     -2.76953125,
     -3.40625,
     -2.51953125,
     -2.005859375,
     -2.0625,
     -2.54296875,
     -2.66796875,
     -1.275390625,
     -2.015625,
     -2.771484375,
     -1.85546875,
     -2.064453125,
     -2.109375,
     -2.08203125,
     -2.03515625,
     -2.125,
     -1.1337890625,
     -1.404296875,
     -2.62890625,
     -1.26953125,
     -2.28125,
     -2.572265625,
     -1.7861328125,
     -2.2109375,
     -2.498046875,
     0.06658935546875,
     -2.31640625,
     -1.7529296875,
     -1.2412109375,
     -0.11883544921875,
     -1.1279296875,
     -3.3125,
     -1.12890625,
     -2.02734375,
     -2.14453125,
     -1.5546875,
     -2.111328125,
     -1.2255859375,
     -1.3544921875,
     -1.0224609375,
     -0.8388671875,
     -0.1019287109375,
     -0.44970703125,
     -0.771484375,
     0.9013671875,
     0.74267578125,
     -0.39111328125,
     -0.036651611328125,
     -0.456298828125,
     0.11053466796875,
     -0.9267578125,
     0.09326171875,
     1.0791015625,
     0.89404296875,
     -0.0767822265625,
     -0.09173583984375,
     -0.7197265625,
     0.303466796875,
     0.54296875,
     -0.71826171875,
     0.053619384765625,
     -0.140380859375,
     0.416748046875,
     0.7802734375,
     -0.76318359375,
     -0.1309814453125,
     2.05859375,
     -0.72607421875,
     -0.27685546875,
     -0.50244140625,
     0.0005345344543457031,
     -0.30810546875,
     -2.701171875
    ],
    "final_raw_logits": [
     0.923828125,
     -4.08984375,
     -1.787109375,
     -3.15234375,
     -3.03515625,
     -3.1484375,
     -3.1015625,
     -3.046875,
     -4.265625,
     -2.974609375,
     -3.037109375,
     -2.724609375,
     -4.38671875,
     -3.40234375,
     -3.67578125,
     -3.583984375,
     -3.9765625,
     -4.8984375,
     -3.580078125,
     -3.421875,
     -3.69140625,
     -2.41015625,
     -5.51171875,
     -5.21875,
     -3.9375,
     -4.7890625,
     -3.427734375,
     -4.73828125,
     -4.14453125,
     -5.67578125,
     -4.3125,
     -4.5390625,
     -5.82421875,
     -3.943359375,
     -3.287109375,
     -4.94140625,
     -5.75,
     -6.21484375,
     -4.6875,
     -3.01953125,
     -5.88671875,
     -4.8046875,
     -5.15234375,
     -5.96484375,
     -3.826171875,
     -4.96484375,
     -4.35546875,
     -4.94921875,
     -5.39453125,
     -4.703125,
     -5.26953125,
     -4.6328125,
     -3.884765625,
     -5.47265625,
     -3.40625,
     -3.783203125,
     -3.955078125,
     -4.4453125,
     -4.58203125,
     -5.77734375,
     -7.453125,
     -7.21484375,
     -8.140625,
     -5.8203125,
     -3.728515625,
     -7.51171875,
     -8.03125,
     -2.712890625,
     -7.87109375,
     -4.35546875,
     -8.0078125,
     -5.390625,
     -2.765625,
     -5.41796875,
     -3.7734375,
     -6.3828125,
     -4.125,
     -7.0390625,
     -4.0390625,
     -6.0703125,
     -4.66015625,
     -4.265625,
     -3.19140625,
     -4.75,
     -3.564453125,
     -4.89453125,
     -2.65625,
     -4.12109375,
     -3.328125,
     -4.48828125,
     -4.02734375,
     -4.14453125,
     -4.31640625,
     -3.615234375,
     -4.78515625,
     -4.1171875,
     -5.1015625,
     -3.16015625,
     -2.66796875,
     -4.76171875,
     -6.18359375,
     -4.96875,
     -3.4140625,
     -4.359375,
     -4.7890625,
     -5.94921875,
     -3.48046875,
     -5.23828125,
     -4.140625,
     -6.0234375,
     -4.1796875,
     -4.98046875,
     -5.04296875,
     -5.79296875,
     -4.703125,
     -3.51171875,
     -4.7890625,
     -6.01953125,
     -4.265625,
     -3.701171875,
     -3.140625,
     -4.2890625,
     -2.89453125,
     -3.689453125,
     -2.212890625,
     -5.1171875,
     -5.84375,
     -2.671875,
     -2.82421875,
     -5.80078125,
     -5.6875,
     -3.255859375,
     -5.8203125,
     -6.65625,
     -5.3671875,
     -4.9609375,
     -3.77734375,
     -4.33984375,
     -6.3671875,
     -2.70703125,
     -3.63671875,
     -4.328125,
     -3.044921875,
     -2.8046875,
     -4.25390625,
     -4.93359375,
     -5.8984375,
     -5.80078125,
     -5.22265625,
     -4.51953125,
     -4.5546875,
     -2.87109375,
     -1.0556640625,
     -3.166015625,
     -1.939453125,
     -1.4638671875,
     -1.576171875,
     -1.51953125,
     -2.634765625,
     -0.87158203125,
     -1.861328125,
     -1.751953125,
     -1.4228515625,
     -1.3427734375,
     -2.6953125,
     -1.84765625,
     -2.994140625,
     -2.28515625,
     -3.892578125,
     -2.1171875,
     -2.21875,
     -2.548828125,
     -1.87890625,
     -2.92578125,
     -0.955078125,
     -1.955078125,
     -3.685546875,
     -3.427734375,
     -3.251953125,
     -2.583984375,
     -3.9296875,
     -2.513671875,
     -3.091796875,
     -2.76953125,
     -3.40625,
     -2.51953125,
     -2.005859375,
     -2.0625,
     -2.54296875,
     -2.66796875,
     -1.275390625,
     -2.015625,
     -2.771484375,
     -1.85546875,
     -2.064453125,
     -2.109375,
     -2.08203125,
     -2.03515625,
     -2.125,
     -1.1337890625,
     -1.404296875,
     -2.62890625,
     -1.26953125,
     -2.28125,
     -2.572265625,
     -1.7861328125,
     -2.2109375,
     -2.498046875,
     0.06658935546875,
     -2.31640625,
     -1.7529296875,
     -1.2412109375,
     -0.11883544921875,
     -1.1279296875,
     -3.3125,
     -1.12890625,
     -2.02734375,
     -2.14453125,
     -1.5546875,
     -2.111328125,
     -1.2255859375,
     -1.3544921875,
     -1.0224609375,
     -0.8388671875,
     -0.1019287109375,
     -0.44970703125,
     -0.771484375,
     0.9013671875,
     0.74267578125,
     -0.39111328125,
     -0.036651611328125,
     -0.456298828125,
     0.11053466796875,
     -0.9267578125,
     0.09326171875,
     1.0791015625,
     0.89404296875,
     -0.0767822265625,
     -0.09173583984375,
     -0.7197265625,
     0.303466796875,
     0.54296875,
     -0.71826171875,
     0.053619384765625,
     -0.140380859375,
     0.416748046875,
     0.7802734375,
     -0.76318359375,
     -0.1309814453125,
     2.05859375,
     -0.72607421875,
     -0.27685546875,
     -0.50244140625,
     0.0005345344543457031,
     -0.30810546875,
     -2.701171875
    ],
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-22T19:27:56+00:00",
   "generated_at": "2026-09-22T19:27:56+00:00",
   "wall_seconds_contended": 16.319394624908455,
   "kind": "kit",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/int8lin-00-kit-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/readout-int8lin-00-kit-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/conversion/slot/readout_gate_slot.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/int8lin-00-kit-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 20.341452957945876,
   "mpsgraph_scratch_before": {
    "at": "2026-09-22T19:27:36+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   },
   "mpsgraph_scratch_after": {
    "at": "2026-09-22T19:27:56+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   }
  },
  {
   "schema": "coreai-slot-readout-session/1",
   "session": "int8lin-01-kit-a1",
   "mode": "int8lin",
   "indices": [
    30,
    31,
    32,
    33,
    34,
    35,
    36,
    37,
    38,
    39,
    40,
    41,
    42,
    43,
    44,
    45,
    46,
    47
   ],
   "pid": 15693,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_int8lin.h16c.aimodelc",
   "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
   "started_at": "2026-09-22T19:28:00+00:00",
   "phase": "complete",
   "runtime_calls": 1274,
   "planned_runtime_calls": 1274,
   "result": "PASS",
   "load_wall_seconds_contended": 0.1865481659770012,
   "active_row": "r11-status",
   "active_step": 36,
   "reset_check": {
    "row": "r11-status",
    "identical": true,
    "max_abs_diff": 0.0,
    "first_raw_logits": [
     -2.8046875,
     3.21875,
     -2.431640625,
     -2.55859375,
     -2.177734375,
     -2.865234375,
     -2.728515625,
     -3.57421875,
     -3.70703125,
     -2.447265625,
     -2.990234375,
     -3.9765625,
     -5.33984375,
     -3.134765625,
     -4.03515625,
     -4.359375,
     -3.697265625,
     -4.31640625,
     -2.919921875,
     -3.05859375,
     -3.41796875,
     -1.994140625,
     -5.27734375,
     -6.87109375,
     -3.90234375,
     -5.359375,
     -2.41015625,
     -5.1953125,
     -4.453125,
     -6.58203125,
     -4.4609375,
     -4.390625,
     -6.1875,
     -4.89453125,
     -3.0625,
     -5.3515625,
     -5.24609375,
     -5.875,
     -4.18359375,
     -2.904296875,
     -7.19140625,
     -4.28125,
     -4.6484375,
     -6.015625,
     -4.25390625,
     -5.04296875,
     -4.5,
     -4.48828125,
     -5.359375,
     -4.65234375,
     -5.67578125,
     -4.68359375,
     -2.73828125,
     -4.86328125,
     -2.654296875,
     -3.333984375,
     -3.533203125,
     -4.40234375,
     -4.70703125,
     -5.43359375,
     -7.54296875,
     -6.3515625,
     -7.62890625,
     -6.5234375,
     -3.61328125,
     -7.4296875,
     -7.67578125,
     -2.3046875,
     -7.7421875,
     -4.07421875,
     -7.78515625,
     -5.28515625,
     -2.474609375,
     -6.1484375,
     -3.724609375,
     -6.1796875,
     -4.2265625,
     -6.67578125,
     -4.23828125,
     -5.65625,
     -4.234375,
     -4.109375,
     -2.66796875,
     -4.21484375,
     -3.51171875,
     -3.90234375,
     -2.583984375,
     -3.80078125,
     -2.9921875,
     -4.84375,
     -3.439453125,
     -4.0625,
     -4.203125,
     -4.19921875,
     -4.58203125,
     -4.65625,
     -5.13671875,
     -3.75,
     -3.9765625,
     -4.14453125,
     -6.046875,
     -4.5078125,
     -3.728515625,
     -4.21484375,
     -4.4765625,
     -5.9765625,
     -4.203125,
     -5.58984375,
     -4.12890625,
     -5.5703125,
     -4.328125,
     -4.59765625,
     -4.94921875,
     -6.4609375,
     -4.74609375,
     -4.015625,
     -4.57421875,
     -6.03125,
     -4.4765625,
     -3.49609375,
     -3.3828125,
     -3.96875,
     -3.275390625,
     -3.783203125,
     -2.37890625,
     -5.828125,
     -5.72265625,
     -3.10546875,
     -3.107421875,
     -5.30859375,
     -5.5546875,
     -3.666015625,
     -5.625,
     -5.953125,
     -4.93359375,
     -4.67578125,
     -3.927734375,
     -4.9453125,
     -5.796875,
     -2.88671875,
     -3.71484375,
     -4.1328125,
     -3.701171875,
     -3.33203125,
     -4.671875,
     -5.19140625,
     -6.08984375,
     -5.140625,
     -4.62109375,
     -4.375,
     -4.734375,
     -3.13671875,
     -1.130859375,
     -3.427734375,
     -2.45703125,
     -1.5966796875,
     -2.01953125,
     -1.92578125,
     -3.412109375,
     -0.4345703125,
     -2.123046875,
     -1.9072265625,
     -1.6650390625,
     -1.2568359375,
     -2.5625,
     -2.302734375,
     -2.865234375,
     -1.76953125,
     -3.4296875,
     -2.021484375,
     -2.296875,
     -2.53515625,
     -2.4375,
     -2.9296875,
     -0.8427734375,
     -1.9345703125,
     -3.19140625,
     -3.28125,
     -3.12890625,
     -3.513671875,
     -3.392578125,
     -3.177734375,
     -2.697265625,
     -2.177734375,
     -2.501953125,
     -1.6376953125,
     -1.326171875,
     -1.75390625,
     -1.919921875,
     -2.0859375,
     -0.9375,
     -1.75,
     -2.849609375,
     -1.701171875,
     -1.390625,
     -2.1484375,
     -2.041015625,
     -1.869140625,
     -1.58984375,
     -1.40234375,
     -1.33984375,
     -1.947265625,
     -1.7197265625,
     -1.89453125,
     -2.263671875,
     -0.96826171875,
     -2.62109375,
     -2.587890625,
     -0.386962890625,
     -2.169921875,
     -0.9130859375,
     -1.2685546875,
     -0.7763671875,
     -1.380859375,
     -2.51953125,
     -1.1875,
     -1.7568359375,
     -2.326171875,
     -1.8173828125,
     -1.9130859375,
     -1.5283203125,
     -1.7099609375,
     -1.462890625,
     -1.330078125,
     -0.21533203125,
     0.107421875,
     -0.48193359375,
     0.83642578125,
     1.076171875,
     -0.03546142578125,
     -0.84326171875,
     -0.480712890625,
     0.034027099609375,
     -0.703125,
     0.55859375,
     1.2607421875,
     1.4638671875,
     -0.7138671875,
     -0.0875244140625,
     -0.90234375,
     0.3818359375,
     0.8291015625,
     -0.8388671875,
     -0.17578125,
     0.04833984375,
     0.4599609375,
     0.315673828125,
     -0.464111328125,
     -0.962890625,
     1.5146484375,
     -0.020111083984375,
     -0.81005859375,
     -0.158935546875,
     0.431640625,
     -0.51806640625,
     -4.2109375
    ],
    "final_raw_logits": [
     -2.8046875,
     3.21875,
     -2.431640625,
     -2.55859375,
     -2.177734375,
     -2.865234375,
     -2.728515625,
     -3.57421875,
     -3.70703125,
     -2.447265625,
     -2.990234375,
     -3.9765625,
     -5.33984375,
     -3.134765625,
     -4.03515625,
     -4.359375,
     -3.697265625,
     -4.31640625,
     -2.919921875,
     -3.05859375,
     -3.41796875,
     -1.994140625,
     -5.27734375,
     -6.87109375,
     -3.90234375,
     -5.359375,
     -2.41015625,
     -5.1953125,
     -4.453125,
     -6.58203125,
     -4.4609375,
     -4.390625,
     -6.1875,
     -4.89453125,
     -3.0625,
     -5.3515625,
     -5.24609375,
     -5.875,
     -4.18359375,
     -2.904296875,
     -7.19140625,
     -4.28125,
     -4.6484375,
     -6.015625,
     -4.25390625,
     -5.04296875,
     -4.5,
     -4.48828125,
     -5.359375,
     -4.65234375,
     -5.67578125,
     -4.68359375,
     -2.73828125,
     -4.86328125,
     -2.654296875,
     -3.333984375,
     -3.533203125,
     -4.40234375,
     -4.70703125,
     -5.43359375,
     -7.54296875,
     -6.3515625,
     -7.62890625,
     -6.5234375,
     -3.61328125,
     -7.4296875,
     -7.67578125,
     -2.3046875,
     -7.7421875,
     -4.07421875,
     -7.78515625,
     -5.28515625,
     -2.474609375,
     -6.1484375,
     -3.724609375,
     -6.1796875,
     -4.2265625,
     -6.67578125,
     -4.23828125,
     -5.65625,
     -4.234375,
     -4.109375,
     -2.66796875,
     -4.21484375,
     -3.51171875,
     -3.90234375,
     -2.583984375,
     -3.80078125,
     -2.9921875,
     -4.84375,
     -3.439453125,
     -4.0625,
     -4.203125,
     -4.19921875,
     -4.58203125,
     -4.65625,
     -5.13671875,
     -3.75,
     -3.9765625,
     -4.14453125,
     -6.046875,
     -4.5078125,
     -3.728515625,
     -4.21484375,
     -4.4765625,
     -5.9765625,
     -4.203125,
     -5.58984375,
     -4.12890625,
     -5.5703125,
     -4.328125,
     -4.59765625,
     -4.94921875,
     -6.4609375,
     -4.74609375,
     -4.015625,
     -4.57421875,
     -6.03125,
     -4.4765625,
     -3.49609375,
     -3.3828125,
     -3.96875,
     -3.275390625,
     -3.783203125,
     -2.37890625,
     -5.828125,
     -5.72265625,
     -3.10546875,
     -3.107421875,
     -5.30859375,
     -5.5546875,
     -3.666015625,
     -5.625,
     -5.953125,
     -4.93359375,
     -4.67578125,
     -3.927734375,
     -4.9453125,
     -5.796875,
     -2.88671875,
     -3.71484375,
     -4.1328125,
     -3.701171875,
     -3.33203125,
     -4.671875,
     -5.19140625,
     -6.08984375,
     -5.140625,
     -4.62109375,
     -4.375,
     -4.734375,
     -3.13671875,
     -1.130859375,
     -3.427734375,
     -2.45703125,
     -1.5966796875,
     -2.01953125,
     -1.92578125,
     -3.412109375,
     -0.4345703125,
     -2.123046875,
     -1.9072265625,
     -1.6650390625,
     -1.2568359375,
     -2.5625,
     -2.302734375,
     -2.865234375,
     -1.76953125,
     -3.4296875,
     -2.021484375,
     -2.296875,
     -2.53515625,
     -2.4375,
     -2.9296875,
     -0.8427734375,
     -1.9345703125,
     -3.19140625,
     -3.28125,
     -3.12890625,
     -3.513671875,
     -3.392578125,
     -3.177734375,
     -2.697265625,
     -2.177734375,
     -2.501953125,
     -1.6376953125,
     -1.326171875,
     -1.75390625,
     -1.919921875,
     -2.0859375,
     -0.9375,
     -1.75,
     -2.849609375,
     -1.701171875,
     -1.390625,
     -2.1484375,
     -2.041015625,
     -1.869140625,
     -1.58984375,
     -1.40234375,
     -1.33984375,
     -1.947265625,
     -1.7197265625,
     -1.89453125,
     -2.263671875,
     -0.96826171875,
     -2.62109375,
     -2.587890625,
     -0.386962890625,
     -2.169921875,
     -0.9130859375,
     -1.2685546875,
     -0.7763671875,
     -1.380859375,
     -2.51953125,
     -1.1875,
     -1.7568359375,
     -2.326171875,
     -1.8173828125,
     -1.9130859375,
     -1.5283203125,
     -1.7099609375,
     -1.462890625,
     -1.330078125,
     -0.21533203125,
     0.107421875,
     -0.48193359375,
     0.83642578125,
     1.076171875,
     -0.03546142578125,
     -0.84326171875,
     -0.480712890625,
     0.034027099609375,
     -0.703125,
     0.55859375,
     1.2607421875,
     1.4638671875,
     -0.7138671875,
     -0.0875244140625,
     -0.90234375,
     0.3818359375,
     0.8291015625,
     -0.8388671875,
     -0.17578125,
     0.04833984375,
     0.4599609375,
     0.315673828125,
     -0.464111328125,
     -0.962890625,
     1.5146484375,
     -0.020111083984375,
     -0.81005859375,
     -0.158935546875,
     0.431640625,
     -0.51806640625,
     -4.2109375
    ],
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-22T19:28:08+00:00",
   "generated_at": "2026-09-22T19:28:08+00:00",
   "wall_seconds_contended": 7.626550000044517,
   "kind": "kit",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/int8lin-01-kit-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/readout-int8lin-01-kit-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/conversion/slot/readout_gate_slot.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/int8lin-01-kit-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 11.66035862499848,
   "mpsgraph_scratch_before": {
    "at": "2026-09-22T19:27:56+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   },
   "mpsgraph_scratch_after": {
    "at": "2026-09-22T19:28:08+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   }
  },
  {
   "schema": "coreai-slot-readout-session/1",
   "session": "int8lin-02-wide-a1",
   "mode": "int8lin",
   "indices": [
    48
   ],
   "pid": 16188,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_int8lin.h16c.aimodelc",
   "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
   "started_at": "2026-09-22T19:28:12+00:00",
   "phase": "complete",
   "runtime_calls": 434,
   "planned_runtime_calls": 434,
   "result": "PASS",
   "load_wall_seconds_contended": 0.17369754193350673,
   "active_row": "r17-slot",
   "active_step": 216,
   "reset_check": {
    "row": "r17-slot",
    "identical": true,
    "max_abs_diff": 0.0,
    "first_raw_logits": [
     -1.87109375,
     -1.8134765625,
     -0.203125,
     -4.3359375,
     -2.033203125,
     -2.892578125,
     -4.45703125,
     -2.681640625,
     -5.171875,
     -2.978515625,
     -3.58203125,
     -3.60546875,
     -6.0546875,
     -5.1796875,
     -4.3203125,
     -3.51953125,
     -3.615234375,
     -5.84765625,
     -3.86328125,
     -4.7734375,
     -5.046875,
     -3.71484375,
     -5.6328125,
     -5.59375,
     -4.1484375,
     -2.25390625,
     -3.25390625,
     5.33203125,
     -4.26171875,
     -6.58984375,
     -3.62109375,
     -7.25390625,
     -4.10546875,
     -5.74609375,
     -4.6015625,
     -5.94921875,
     -7.2578125,
     -7.40234375,
     -7.125,
     -4.06640625,
     -6.56640625,
     -6.234375,
     -6.12890625,
     -6.8359375,
     -4.921875,
     -6.26171875,
     -4.70703125,
     -4.56640625,
     -5.54296875,
     -6.2890625,
     -4.8046875,
     -5.7421875,
     -5.11328125,
     -4.59765625,
     -4.65234375,
     -4.66015625,
     -4.6640625,
     -4.734375,
     -5.3515625,
     -6.609375,
     -8.0703125,
     -7.546875,
     -9.0,
     -6.61328125,
     -4.70703125,
     -8.6640625,
     -8.96875,
     -2.57421875,
     -9.9921875,
     -5.30859375,
     -8.6640625,
     -5.8828125,
     -2.8515625,
     -7.2109375,
     -3.705078125,
     -6.765625,
     -4.6328125,
     -7.78515625,
     -4.36328125,
     -6.03125,
     -3.59375,
     -4.921875,
     -3.646484375,
     -4.62890625,
     -4.21484375,
     -5.4453125,
     -3.30859375,
     -4.95703125,
     -3.697265625,
     -5.58203125,
     -4.98046875,
     -3.849609375,
     -4.0859375,
     -5.16015625,
     -5.578125,
     -5.6640625,
     -5.98046875,
     -3.5390625,
     -3.42578125,
     -5.8203125,
     -7.0,
     -5.0234375,
     -3.57421875,
     -5.74609375,
     -5.51953125,
     -7.27734375,
     -4.41796875,
     -6.0546875,
     -4.45703125,
     -7.4765625,
     -5.28515625,
     -5.7421875,
     -5.98046875,
     -6.52734375,
     -5.2109375,
     -2.845703125,
     -5.08984375,
     -7.0,
     -5.578125,
     -4.1640625,
     -4.30859375,
     -4.234375,
     -3.126953125,
     -3.29296875,
     -2.716796875,
     -6.22265625,
     -7.16015625,
     -3.240234375,
     -2.974609375,
     -6.0546875,
     -5.23828125,
     -2.6171875,
     -6.46875,
     -7.03515625,
     -5.46875,
     -5.6640625,
     -3.86328125,
     -5.26171875,
     -7.56640625,
     -3.513671875,
     -3.1328125,
     -4.33984375,
     -4.03125,
     -2.642578125,
     -4.1171875,
     -4.75390625,
     -6.92578125,
     -6.53515625,
     -6.15625,
     -5.0859375,
     -4.51953125,
     -3.283203125,
     -1.2998046875,
     -2.677734375,
     -1.7119140625,
     -2.01171875,
     -1.94921875,
     -1.2333984375,
     -3.115234375,
     0.327392578125,
     -1.328125,
     -1.6298828125,
     -1.1005859375,
     -1.7978515625,
     -2.45703125,
     -1.7646484375,
     -3.537109375,
     -1.7451171875,
     -5.19921875,
     -2.67578125,
     -3.5234375,
     -2.314453125,
     -1.99609375,
     -3.65234375,
     -1.5166015625,
     -3.353515625,
     -3.25390625,
     -3.408203125,
     -3.326171875,
     -4.28125,
     -4.19140625,
     -2.74609375,
     -2.515625,
     -3.0859375,
     -2.888671875,
     -2.611328125,
     -2.970703125,
     -1.822265625,
     -3.294921875,
     -2.658203125,
     -2.490234375,
     -2.265625,
     -3.486328125,
     -2.857421875,
     -1.8173828125,
     -1.701171875,
     -1.8505859375,
     -2.19921875,
     -2.16015625,
     -1.0712890625,
     -1.15625,
     -3.20703125,
     -1.609375,
     -1.791015625,
     -2.82421875,
     -1.560546875,
     -2.330078125,
     -3.060546875,
     0.189208984375,
     -2.525390625,
     -1.0009765625,
     -1.267578125,
     -1.0986328125,
     -1.9990234375,
     -4.06640625,
     -2.0078125,
     -1.328125,
     -2.4140625,
     -1.2333984375,
     -2.01953125,
     -1.7783203125,
     -2.248046875,
     -2.220703125,
     -0.96875,
     0.461669921875,
     -0.57958984375,
     -0.0936279296875,
     0.358642578125,
     0.892578125,
     -1.244140625,
     0.258056640625,
     0.261962890625,
     0.050048828125,
     -0.7080078125,
     0.155517578125,
     1.37109375,
     1.3359375,
     -0.234619140625,
     0.41015625,
     -1.00390625,
     0.28564453125,
     0.6328125,
     -0.19580078125,
     -0.00981903076171875,
     0.182373046875,
     -0.01959228515625,
     0.4248046875,
     -0.03143310546875,
     -1.28515625,
     2.16015625,
     -0.82958984375,
     0.6435546875,
     -0.475830078125,
     0.099365234375,
     0.1785888671875,
     -4.46484375
    ],
    "final_raw_logits": [
     -1.87109375,
     -1.8134765625,
     -0.203125,
     -4.3359375,
     -2.033203125,
     -2.892578125,
     -4.45703125,
     -2.681640625,
     -5.171875,
     -2.978515625,
     -3.58203125,
     -3.60546875,
     -6.0546875,
     -5.1796875,
     -4.3203125,
     -3.51953125,
     -3.615234375,
     -5.84765625,
     -3.86328125,
     -4.7734375,
     -5.046875,
     -3.71484375,
     -5.6328125,
     -5.59375,
     -4.1484375,
     -2.25390625,
     -3.25390625,
     5.33203125,
     -4.26171875,
     -6.58984375,
     -3.62109375,
     -7.25390625,
     -4.10546875,
     -5.74609375,
     -4.6015625,
     -5.94921875,
     -7.2578125,
     -7.40234375,
     -7.125,
     -4.06640625,
     -6.56640625,
     -6.234375,
     -6.12890625,
     -6.8359375,
     -4.921875,
     -6.26171875,
     -4.70703125,
     -4.56640625,
     -5.54296875,
     -6.2890625,
     -4.8046875,
     -5.7421875,
     -5.11328125,
     -4.59765625,
     -4.65234375,
     -4.66015625,
     -4.6640625,
     -4.734375,
     -5.3515625,
     -6.609375,
     -8.0703125,
     -7.546875,
     -9.0,
     -6.61328125,
     -4.70703125,
     -8.6640625,
     -8.96875,
     -2.57421875,
     -9.9921875,
     -5.30859375,
     -8.6640625,
     -5.8828125,
     -2.8515625,
     -7.2109375,
     -3.705078125,
     -6.765625,
     -4.6328125,
     -7.78515625,
     -4.36328125,
     -6.03125,
     -3.59375,
     -4.921875,
     -3.646484375,
     -4.62890625,
     -4.21484375,
     -5.4453125,
     -3.30859375,
     -4.95703125,
     -3.697265625,
     -5.58203125,
     -4.98046875,
     -3.849609375,
     -4.0859375,
     -5.16015625,
     -5.578125,
     -5.6640625,
     -5.98046875,
     -3.5390625,
     -3.42578125,
     -5.8203125,
     -7.0,
     -5.0234375,
     -3.57421875,
     -5.74609375,
     -5.51953125,
     -7.27734375,
     -4.41796875,
     -6.0546875,
     -4.45703125,
     -7.4765625,
     -5.28515625,
     -5.7421875,
     -5.98046875,
     -6.52734375,
     -5.2109375,
     -2.845703125,
     -5.08984375,
     -7.0,
     -5.578125,
     -4.1640625,
     -4.30859375,
     -4.234375,
     -3.126953125,
     -3.29296875,
     -2.716796875,
     -6.22265625,
     -7.16015625,
     -3.240234375,
     -2.974609375,
     -6.0546875,
     -5.23828125,
     -2.6171875,
     -6.46875,
     -7.03515625,
     -5.46875,
     -5.6640625,
     -3.86328125,
     -5.26171875,
     -7.56640625,
     -3.513671875,
     -3.1328125,
     -4.33984375,
     -4.03125,
     -2.642578125,
     -4.1171875,
     -4.75390625,
     -6.92578125,
     -6.53515625,
     -6.15625,
     -5.0859375,
     -4.51953125,
     -3.283203125,
     -1.2998046875,
     -2.677734375,
     -1.7119140625,
     -2.01171875,
     -1.94921875,
     -1.2333984375,
     -3.115234375,
     0.327392578125,
     -1.328125,
     -1.6298828125,
     -1.1005859375,
     -1.7978515625,
     -2.45703125,
     -1.7646484375,
     -3.537109375,
     -1.7451171875,
     -5.19921875,
     -2.67578125,
     -3.5234375,
     -2.314453125,
     -1.99609375,
     -3.65234375,
     -1.5166015625,
     -3.353515625,
     -3.25390625,
     -3.408203125,
     -3.326171875,
     -4.28125,
     -4.19140625,
     -2.74609375,
     -2.515625,
     -3.0859375,
     -2.888671875,
     -2.611328125,
     -2.970703125,
     -1.822265625,
     -3.294921875,
     -2.658203125,
     -2.490234375,
     -2.265625,
     -3.486328125,
     -2.857421875,
     -1.8173828125,
     -1.701171875,
     -1.8505859375,
     -2.19921875,
     -2.16015625,
     -1.0712890625,
     -1.15625,
     -3.20703125,
     -1.609375,
     -1.791015625,
     -2.82421875,
     -1.560546875,
     -2.330078125,
     -3.060546875,
     0.189208984375,
     -2.525390625,
     -1.0009765625,
     -1.267578125,
     -1.0986328125,
     -1.9990234375,
     -4.06640625,
     -2.0078125,
     -1.328125,
     -2.4140625,
     -1.2333984375,
     -2.01953125,
     -1.7783203125,
     -2.248046875,
     -2.220703125,
     -0.96875,
     0.461669921875,
     -0.57958984375,
     -0.0936279296875,
     0.358642578125,
     0.892578125,
     -1.244140625,
     0.258056640625,
     0.261962890625,
     0.050048828125,
     -0.7080078125,
     0.155517578125,
     1.37109375,
     1.3359375,
     -0.234619140625,
     0.41015625,
     -1.00390625,
     0.28564453125,
     0.6328125,
     -0.19580078125,
     -0.00981903076171875,
     0.182373046875,
     -0.01959228515625,
     0.4248046875,
     -0.03143310546875,
     -1.28515625,
     2.16015625,
     -0.82958984375,
     0.6435546875,
     -0.475830078125,
     0.099365234375,
     0.1785888671875,
     -4.46484375
    ],
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-22T19:28:15+00:00",
   "generated_at": "2026-09-22T19:28:15+00:00",
   "wall_seconds_contended": 2.7757141669280827,
   "kind": "wide",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/int8lin-02-wide-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/readout-int8lin-02-wide-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/conversion/slot/readout_gate_slot.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/int8lin-02-wide-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 6.812467582989484,
   "mpsgraph_scratch_before": {
    "at": "2026-09-22T19:28:08+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   },
   "mpsgraph_scratch_after": {
    "at": "2026-09-22T19:28:15+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   }
  },
  {
   "schema": "coreai-slot-readout-session/1",
   "session": "int8lin-03-wide-a1",
   "mode": "int8lin",
   "indices": [
    49
   ],
   "pid": 16474,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_int8lin.h16c.aimodelc",
   "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
   "started_at": "2026-09-22T19:28:19+00:00",
   "phase": "complete",
   "runtime_calls": 2898,
   "planned_runtime_calls": 2898,
   "result": "PASS",
   "load_wall_seconds_contended": 0.1803588749608025,
   "active_row": "r18-slot",
   "active_step": 1448,
   "reset_check": {
    "row": "r18-slot",
    "identical": true,
    "max_abs_diff": 0.0,
    "first_raw_logits": [
     -5.0859375,
     -5.1015625,
     -7.3046875,
     -6.6328125,
     -7.515625,
     -8.1171875,
     -7.09765625,
     -8.4140625,
     -9.453125,
     -6.34375,
     -9.140625,
     -9.25,
     -7.39453125,
     -9.4296875,
     -9.078125,
     -8.0859375,
     -10.1484375,
     -9.3671875,
     -7.83203125,
     -10.1875,
     -9.34375,
     -7.8671875,
     -10.3203125,
     -10.109375,
     -11.1796875,
     -12.0,
     -10.3984375,
     -11.1328125,
     -10.1328125,
     -10.3515625,
     -9.515625,
     -13.2890625,
     -11.375,
     -10.3046875,
     -9.765625,
     -11.5859375,
     -11.3515625,
     -9.1875,
     -9.578125,
     -7.71484375,
     -11.09375,
     -11.0234375,
     -12.375,
     -10.203125,
     -10.3515625,
     -11.9140625,
     -9.953125,
     -9.8828125,
     -10.609375,
     -10.6640625,
     -13.078125,
     -11.5,
     -12.3984375,
     -11.1015625,
     -10.9140625,
     -9.2890625,
     -9.40625,
     -8.21875,
     -9.203125,
     -7.03125,
     -14.796875,
     -13.7109375,
     -14.921875,
     -11.0390625,
     -11.0546875,
     -18.140625,
     -16.796875,
     -8.9765625,
     -17.21875,
     -10.9375,
     -16.125,
     -10.0703125,
     -7.26171875,
     -12.0,
     -9.1328125,
     -15.0,
     -10.046875,
     -14.3515625,
     -8.3984375,
     -12.046875,
     -8.6875,
     -10.78125,
     -7.0078125,
     -12.5546875,
     -8.671875,
     -9.9765625,
     -8.578125,
     -9.90625,
     -9.390625,
     -14.5078125,
     -11.640625,
     -13.1015625,
     -10.140625,
     -10.0703125,
     -9.6953125,
     -11.5,
     -9.71875,
     -7.0546875,
     -11.859375,
     -9.8671875,
     -16.03125,
     -8.6484375,
     -8.78125,
     -10.765625,
     -8.7421875,
     -15.8203125,
     -11.0078125,
     -12.3671875,
     -9.828125,
     -14.2890625,
     -7.98046875,
     -14.2734375,
     -14.390625,
     -15.5625,
     -10.296875,
     -6.65234375,
     -10.1171875,
     -15.21875,
     -12.875,
     -10.140625,
     -8.15625,
     -7.0,
     -7.98046875,
     -9.375,
     -7.27734375,
     -14.9296875,
     -15.421875,
     -9.609375,
     -8.84375,
     -16.4375,
     -12.0546875,
     -11.671875,
     -16.171875,
     -16.0,
     -8.1953125,
     -15.2421875,
     -10.65625,
     -10.2734375,
     -16.203125,
     -12.625,
     -9.7734375,
     -11.609375,
     -10.015625,
     -9.9375,
     -9.1796875,
     -10.4140625,
     -16.203125,
     -14.390625,
     -11.328125,
     -10.3828125,
     -11.6328125,
     -10.6484375,
     -10.796875,
     -10.1640625,
     -5.9609375,
     -5.78125,
     -4.89453125,
     -5.34765625,
     -7.4609375,
     -9.0,
     -6.65234375,
     -6.421875,
     -6.29296875,
     -6.29296875,
     -9.5,
     -3.615234375,
     -7.05078125,
     -7.1953125,
     -10.8515625,
     -8.1796875,
     -7.55859375,
     -7.5234375,
     -4.46484375,
     -9.234375,
     -7.38671875,
     -7.5,
     -10.9296875,
     -9.671875,
     -11.3203125,
     -10.0859375,
     -10.921875,
     -12.8515625,
     -10.0625,
     -10.734375,
     -11.453125,
     -9.7578125,
     -9.9296875,
     -10.0859375,
     -11.03125,
     -11.6328125,
     -11.34375,
     -9.8203125,
     -10.6875,
     -9.4609375,
     -11.3125,
     -10.3515625,
     -11.3515625,
     -9.3203125,
     -9.7734375,
     -10.1328125,
     -9.6015625,
     -10.5234375,
     -8.3046875,
     -11.046875,
     -9.7109375,
     -8.7578125,
     -10.171875,
     -8.796875,
     -0.72119140625,
     -10.71875,
     -8.859375,
     -9.7890625,
     -9.3671875,
     -9.0703125,
     -9.3359375,
     -9.046875,
     -10.265625,
     -8.5078125,
     -8.984375,
     -5.8828125,
     -7.05859375,
     -0.03045654296875,
     -1.78125,
     -1.072265625,
     1.5087890625,
     -0.83447265625,
     0.1378173828125,
     -0.45263671875,
     -0.7734375,
     0.2568359375,
     0.8896484375,
     1.4951171875,
     -1.9375,
     -0.1380615234375,
     1.181640625,
     0.74462890625,
     0.370361328125,
     -0.57763671875,
     0.86962890625,
     0.382080078125,
     -0.036529541015625,
     -1.2607421875,
     -1.34765625,
     1.1630859375,
     -0.26513671875,
     1.732421875,
     -1.1767578125,
     0.1639404296875,
     -1.3857421875,
     -0.2169189453125,
     -1.4521484375,
     1.1376953125,
     -0.2081298828125,
     1.818359375,
     -1.2392578125,
     -4.66796875
    ],
    "final_raw_logits": [
     -5.0859375,
     -5.1015625,
     -7.3046875,
     -6.6328125,
     -7.515625,
     -8.1171875,
     -7.09765625,
     -8.4140625,
     -9.453125,
     -6.34375,
     -9.140625,
     -9.25,
     -7.39453125,
     -9.4296875,
     -9.078125,
     -8.0859375,
     -10.1484375,
     -9.3671875,
     -7.83203125,
     -10.1875,
     -9.34375,
     -7.8671875,
     -10.3203125,
     -10.109375,
     -11.1796875,
     -12.0,
     -10.3984375,
     -11.1328125,
     -10.1328125,
     -10.3515625,
     -9.515625,
     -13.2890625,
     -11.375,
     -10.3046875,
     -9.765625,
     -11.5859375,
     -11.3515625,
     -9.1875,
     -9.578125,
     -7.71484375,
     -11.09375,
     -11.0234375,
     -12.375,
     -10.203125,
     -10.3515625,
     -11.9140625,
     -9.953125,
     -9.8828125,
     -10.609375,
     -10.6640625,
     -13.078125,
     -11.5,
     -12.3984375,
     -11.1015625,
     -10.9140625,
     -9.2890625,
     -9.40625,
     -8.21875,
     -9.203125,
     -7.03125,
     -14.796875,
     -13.7109375,
     -14.921875,
     -11.0390625,
     -11.0546875,
     -18.140625,
     -16.796875,
     -8.9765625,
     -17.21875,
     -10.9375,
     -16.125,
     -10.0703125,
     -7.26171875,
     -12.0,
     -9.1328125,
     -15.0,
     -10.046875,
     -14.3515625,
     -8.3984375,
     -12.046875,
     -8.6875,
     -10.78125,
     -7.0078125,
     -12.5546875,
     -8.671875,
     -9.9765625,
     -8.578125,
     -9.90625,
     -9.390625,
     -14.5078125,
     -11.640625,
     -13.1015625,
     -10.140625,
     -10.0703125,
     -9.6953125,
     -11.5,
     -9.71875,
     -7.0546875,
     -11.859375,
     -9.8671875,
     -16.03125,
     -8.6484375,
     -8.78125,
     -10.765625,
     -8.7421875,
     -15.8203125,
     -11.0078125,
     -12.3671875,
     -9.828125,
     -14.2890625,
     -7.98046875,
     -14.2734375,
     -14.390625,
     -15.5625,
     -10.296875,
     -6.65234375,
     -10.1171875,
     -15.21875,
     -12.875,
     -10.140625,
     -8.15625,
     -7.0,
     -7.98046875,
     -9.375,
     -7.27734375,
     -14.9296875,
     -15.421875,
     -9.609375,
     -8.84375,
     -16.4375,
     -12.0546875,
     -11.671875,
     -16.171875,
     -16.0,
     -8.1953125,
     -15.2421875,
     -10.65625,
     -10.2734375,
     -16.203125,
     -12.625,
     -9.7734375,
     -11.609375,
     -10.015625,
     -9.9375,
     -9.1796875,
     -10.4140625,
     -16.203125,
     -14.390625,
     -11.328125,
     -10.3828125,
     -11.6328125,
     -10.6484375,
     -10.796875,
     -10.1640625,
     -5.9609375,
     -5.78125,
     -4.89453125,
     -5.34765625,
     -7.4609375,
     -9.0,
     -6.65234375,
     -6.421875,
     -6.29296875,
     -6.29296875,
     -9.5,
     -3.615234375,
     -7.05078125,
     -7.1953125,
     -10.8515625,
     -8.1796875,
     -7.55859375,
     -7.5234375,
     -4.46484375,
     -9.234375,
     -7.38671875,
     -7.5,
     -10.9296875,
     -9.671875,
     -11.3203125,
     -10.0859375,
     -10.921875,
     -12.8515625,
     -10.0625,
     -10.734375,
     -11.453125,
     -9.7578125,
     -9.9296875,
     -10.0859375,
     -11.03125,
     -11.6328125,
     -11.34375,
     -9.8203125,
     -10.6875,
     -9.4609375,
     -11.3125,
     -10.3515625,
     -11.3515625,
     -9.3203125,
     -9.7734375,
     -10.1328125,
     -9.6015625,
     -10.5234375,
     -8.3046875,
     -11.046875,
     -9.7109375,
     -8.7578125,
     -10.171875,
     -8.796875,
     -0.72119140625,
     -10.71875,
     -8.859375,
     -9.7890625,
     -9.3671875,
     -9.0703125,
     -9.3359375,
     -9.046875,
     -10.265625,
     -8.5078125,
     -8.984375,
     -5.8828125,
     -7.05859375,
     -0.03045654296875,
     -1.78125,
     -1.072265625,
     1.5087890625,
     -0.83447265625,
     0.1378173828125,
     -0.45263671875,
     -0.7734375,
     0.2568359375,
     0.8896484375,
     1.4951171875,
     -1.9375,
     -0.1380615234375,
     1.181640625,
     0.74462890625,
     0.370361328125,
     -0.57763671875,
     0.86962890625,
     0.382080078125,
     -0.036529541015625,
     -1.2607421875,
     -1.34765625,
     1.1630859375,
     -0.26513671875,
     1.732421875,
     -1.1767578125,
     0.1639404296875,
     -1.3857421875,
     -0.2169189453125,
     -1.4521484375,
     1.1376953125,
     -0.2081298828125,
     1.818359375,
     -1.2392578125,
     -4.66796875
    ],
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-22T19:28:35+00:00",
   "generated_at": "2026-09-22T19:28:35+00:00",
   "wall_seconds_contended": 16.6014877500711,
   "kind": "wide",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/int8lin-03-wide-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/readout-int8lin-03-wide-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/conversion/slot/readout_gate_slot.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/int8lin-03-wide-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 20.63658349995967,
   "mpsgraph_scratch_before": {
    "at": "2026-09-22T19:28:15+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   },
   "mpsgraph_scratch_after": {
    "at": "2026-09-22T19:28:35+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   }
  },
  {
   "schema": "coreai-slot-readout-session/1",
   "session": "int8lin-04-global_reset-a1",
   "mode": "int8lin",
   "indices": [
    0
   ],
   "pid": 17296,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_int8lin.h16c.aimodelc",
   "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
   "started_at": "2026-09-22T19:28:39+00:00",
   "phase": "complete",
   "runtime_calls": 226,
   "planned_runtime_calls": 226,
   "result": "PASS",
   "load_wall_seconds_contended": 0.19715679192449898,
   "active_row": "r01-route",
   "active_step": 112,
   "reset_check": {
    "row": "r01-route",
    "identical": true,
    "max_abs_diff": 0.0,
    "first_raw_logits": [
     0.923828125,
     -4.08984375,
     -1.787109375,
     -3.15234375,
     -3.03515625,
     -3.1484375,
     -3.1015625,
     -3.046875,
     -4.265625,
     -2.974609375,
     -3.037109375,
     -2.724609375,
     -4.38671875,
     -3.40234375,
     -3.67578125,
     -3.583984375,
     -3.9765625,
     -4.8984375,
     -3.580078125,
     -3.421875,
     -3.69140625,
     -2.41015625,
     -5.51171875,
     -5.21875,
     -3.9375,
     -4.7890625,
     -3.427734375,
     -4.73828125,
     -4.14453125,
     -5.67578125,
     -4.3125,
     -4.5390625,
     -5.82421875,
     -3.943359375,
     -3.287109375,
     -4.94140625,
     -5.75,
     -6.21484375,
     -4.6875,
     -3.01953125,
     -5.88671875,
     -4.8046875,
     -5.15234375,
     -5.96484375,
     -3.826171875,
     -4.96484375,
     -4.35546875,
     -4.94921875,
     -5.39453125,
     -4.703125,
     -5.26953125,
     -4.6328125,
     -3.884765625,
     -5.47265625,
     -3.40625,
     -3.783203125,
     -3.955078125,
     -4.4453125,
     -4.58203125,
     -5.77734375,
     -7.453125,
     -7.21484375,
     -8.140625,
     -5.8203125,
     -3.728515625,
     -7.51171875,
     -8.03125,
     -2.712890625,
     -7.87109375,
     -4.35546875,
     -8.0078125,
     -5.390625,
     -2.765625,
     -5.41796875,
     -3.7734375,
     -6.3828125,
     -4.125,
     -7.0390625,
     -4.0390625,
     -6.0703125,
     -4.66015625,
     -4.265625,
     -3.19140625,
     -4.75,
     -3.564453125,
     -4.89453125,
     -2.65625,
     -4.12109375,
     -3.328125,
     -4.48828125,
     -4.02734375,
     -4.14453125,
     -4.31640625,
     -3.615234375,
     -4.78515625,
     -4.1171875,
     -5.1015625,
     -3.16015625,
     -2.66796875,
     -4.76171875,
     -6.18359375,
     -4.96875,
     -3.4140625,
     -4.359375,
     -4.7890625,
     -5.94921875,
     -3.48046875,
     -5.23828125,
     -4.140625,
     -6.0234375,
     -4.1796875,
     -4.98046875,
     -5.04296875,
     -5.79296875,
     -4.703125,
     -3.51171875,
     -4.7890625,
     -6.01953125,
     -4.265625,
     -3.701171875,
     -3.140625,
     -4.2890625,
     -2.89453125,
     -3.689453125,
     -2.212890625,
     -5.1171875,
     -5.84375,
     -2.671875,
     -2.82421875,
     -5.80078125,
     -5.6875,
     -3.255859375,
     -5.8203125,
     -6.65625,
     -5.3671875,
     -4.9609375,
     -3.77734375,
     -4.33984375,
     -6.3671875,
     -2.70703125,
     -3.63671875,
     -4.328125,
     -3.044921875,
     -2.8046875,
     -4.25390625,
     -4.93359375,
     -5.8984375,
     -5.80078125,
     -5.22265625,
     -4.51953125,
     -4.5546875,
     -2.87109375,
     -1.0556640625,
     -3.166015625,
     -1.939453125,
     -1.4638671875,
     -1.576171875,
     -1.51953125,
     -2.634765625,
     -0.87158203125,
     -1.861328125,
     -1.751953125,
     -1.4228515625,
     -1.3427734375,
     -2.6953125,
     -1.84765625,
     -2.994140625,
     -2.28515625,
     -3.892578125,
     -2.1171875,
     -2.21875,
     -2.548828125,
     -1.87890625,
     -2.92578125,
     -0.955078125,
     -1.955078125,
     -3.685546875,
     -3.427734375,
     -3.251953125,
     -2.583984375,
     -3.9296875,
     -2.513671875,
     -3.091796875,
     -2.76953125,
     -3.40625,
     -2.51953125,
     -2.005859375,
     -2.0625,
     -2.54296875,
     -2.66796875,
     -1.275390625,
     -2.015625,
     -2.771484375,
     -1.85546875,
     -2.064453125,
     -2.109375,
     -2.08203125,
     -2.03515625,
     -2.125,
     -1.1337890625,
     -1.404296875,
     -2.62890625,
     -1.26953125,
     -2.28125,
     -2.572265625,
     -1.7861328125,
     -2.2109375,
     -2.498046875,
     0.06658935546875,
     -2.31640625,
     -1.7529296875,
     -1.2412109375,
     -0.11883544921875,
     -1.1279296875,
     -3.3125,
     -1.12890625,
     -2.02734375,
     -2.14453125,
     -1.5546875,
     -2.111328125,
     -1.2255859375,
     -1.3544921875,
     -1.0224609375,
     -0.8388671875,
     -0.1019287109375,
     -0.44970703125,
     -0.771484375,
     0.9013671875,
     0.74267578125,
     -0.39111328125,
     -0.036651611328125,
     -0.456298828125,
     0.11053466796875,
     -0.9267578125,
     0.09326171875,
     1.0791015625,
     0.89404296875,
     -0.0767822265625,
     -0.09173583984375,
     -0.7197265625,
     0.303466796875,
     0.54296875,
     -0.71826171875,
     0.053619384765625,
     -0.140380859375,
     0.416748046875,
     0.7802734375,
     -0.76318359375,
     -0.1309814453125,
     2.05859375,
     -0.72607421875,
     -0.27685546875,
     -0.50244140625,
     0.0005345344543457031,
     -0.30810546875,
     -2.701171875
    ],
    "final_raw_logits": [
     0.923828125,
     -4.08984375,
     -1.787109375,
     -3.15234375,
     -3.03515625,
     -3.1484375,
     -3.1015625,
     -3.046875,
     -4.265625,
     -2.974609375,
     -3.037109375,
     -2.724609375,
     -4.38671875,
     -3.40234375,
     -3.67578125,
     -3.583984375,
     -3.9765625,
     -4.8984375,
     -3.580078125,
     -3.421875,
     -3.69140625,
     -2.41015625,
     -5.51171875,
     -5.21875,
     -3.9375,
     -4.7890625,
     -3.427734375,
     -4.73828125,
     -4.14453125,
     -5.67578125,
     -4.3125,
     -4.5390625,
     -5.82421875,
     -3.943359375,
     -3.287109375,
     -4.94140625,
     -5.75,
     -6.21484375,
     -4.6875,
     -3.01953125,
     -5.88671875,
     -4.8046875,
     -5.15234375,
     -5.96484375,
     -3.826171875,
     -4.96484375,
     -4.35546875,
     -4.94921875,
     -5.39453125,
     -4.703125,
     -5.26953125,
     -4.6328125,
     -3.884765625,
     -5.47265625,
     -3.40625,
     -3.783203125,
     -3.955078125,
     -4.4453125,
     -4.58203125,
     -5.77734375,
     -7.453125,
     -7.21484375,
     -8.140625,
     -5.8203125,
     -3.728515625,
     -7.51171875,
     -8.03125,
     -2.712890625,
     -7.87109375,
     -4.35546875,
     -8.0078125,
     -5.390625,
     -2.765625,
     -5.41796875,
     -3.7734375,
     -6.3828125,
     -4.125,
     -7.0390625,
     -4.0390625,
     -6.0703125,
     -4.66015625,
     -4.265625,
     -3.19140625,
     -4.75,
     -3.564453125,
     -4.89453125,
     -2.65625,
     -4.12109375,
     -3.328125,
     -4.48828125,
     -4.02734375,
     -4.14453125,
     -4.31640625,
     -3.615234375,
     -4.78515625,
     -4.1171875,
     -5.1015625,
     -3.16015625,
     -2.66796875,
     -4.76171875,
     -6.18359375,
     -4.96875,
     -3.4140625,
     -4.359375,
     -4.7890625,
     -5.94921875,
     -3.48046875,
     -5.23828125,
     -4.140625,
     -6.0234375,
     -4.1796875,
     -4.98046875,
     -5.04296875,
     -5.79296875,
     -4.703125,
     -3.51171875,
     -4.7890625,
     -6.01953125,
     -4.265625,
     -3.701171875,
     -3.140625,
     -4.2890625,
     -2.89453125,
     -3.689453125,
     -2.212890625,
     -5.1171875,
     -5.84375,
     -2.671875,
     -2.82421875,
     -5.80078125,
     -5.6875,
     -3.255859375,
     -5.8203125,
     -6.65625,
     -5.3671875,
     -4.9609375,
     -3.77734375,
     -4.33984375,
     -6.3671875,
     -2.70703125,
     -3.63671875,
     -4.328125,
     -3.044921875,
     -2.8046875,
     -4.25390625,
     -4.93359375,
     -5.8984375,
     -5.80078125,
     -5.22265625,
     -4.51953125,
     -4.5546875,
     -2.87109375,
     -1.0556640625,
     -3.166015625,
     -1.939453125,
     -1.4638671875,
     -1.576171875,
     -1.51953125,
     -2.634765625,
     -0.87158203125,
     -1.861328125,
     -1.751953125,
     -1.4228515625,
     -1.3427734375,
     -2.6953125,
     -1.84765625,
     -2.994140625,
     -2.28515625,
     -3.892578125,
     -2.1171875,
     -2.21875,
     -2.548828125,
     -1.87890625,
     -2.92578125,
     -0.955078125,
     -1.955078125,
     -3.685546875,
     -3.427734375,
     -3.251953125,
     -2.583984375,
     -3.9296875,
     -2.513671875,
     -3.091796875,
     -2.76953125,
     -3.40625,
     -2.51953125,
     -2.005859375,
     -2.0625,
     -2.54296875,
     -2.66796875,
     -1.275390625,
     -2.015625,
     -2.771484375,
     -1.85546875,
     -2.064453125,
     -2.109375,
     -2.08203125,
     -2.03515625,
     -2.125,
     -1.1337890625,
     -1.404296875,
     -2.62890625,
     -1.26953125,
     -2.28125,
     -2.572265625,
     -1.7861328125,
     -2.2109375,
     -2.498046875,
     0.06658935546875,
     -2.31640625,
     -1.7529296875,
     -1.2412109375,
     -0.11883544921875,
     -1.1279296875,
     -3.3125,
     -1.12890625,
     -2.02734375,
     -2.14453125,
     -1.5546875,
     -2.111328125,
     -1.2255859375,
     -1.3544921875,
     -1.0224609375,
     -0.8388671875,
     -0.1019287109375,
     -0.44970703125,
     -0.771484375,
     0.9013671875,
     0.74267578125,
     -0.39111328125,
     -0.036651611328125,
     -0.456298828125,
     0.11053466796875,
     -0.9267578125,
     0.09326171875,
     1.0791015625,
     0.89404296875,
     -0.0767822265625,
     -0.09173583984375,
     -0.7197265625,
     0.303466796875,
     0.54296875,
     -0.71826171875,
     0.053619384765625,
     -0.140380859375,
     0.416748046875,
     0.7802734375,
     -0.76318359375,
     -0.1309814453125,
     2.05859375,
     -0.72607421875,
     -0.27685546875,
     -0.50244140625,
     0.0005345344543457031,
     -0.30810546875,
     -2.701171875
    ],
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-22T19:28:41+00:00",
   "generated_at": "2026-09-22T19:28:41+00:00",
   "wall_seconds_contended": 1.6544022499583662,
   "kind": "global_reset",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/int8lin-04-global_reset-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/readout-int8lin-04-global_reset-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/conversion/slot/readout_gate_slot.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-int8lin/int8lin-04-global_reset-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 5.700364583986811,
   "mpsgraph_scratch_before": {
    "at": "2026-09-22T19:28:35+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   },
   "mpsgraph_scratch_after": {
    "at": "2026-09-22T19:28:41+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   }
  }
 ],
 "result": "PASS",
 "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_int8lin.h16c.aimodelc",
 "aot": {
  "source": {
   "path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/openthai_systemone_decode_int8lin/openthai_systemone_decode_int8lin.aimodel",
   "bytes": 160,
   "mtime_ns": 1790102235022015681
  },
  "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_int8lin.h16c.aimodelc",
  "result": "PASS",
  "reused": true,
  "wall_seconds": 0.0,
  "note": "Existing explicitly selected AOT asset reused; no GPU JIT."
 },
 "aot_seconds": 0.0,
 "summary": {
  "rows": 50,
  "expected_rows": 50,
  "full_vocab_argmax_is_label": 43,
  "argmax_agreement": 50,
  "margin_ge_0_02_rows": 49,
  "margin_ge_0_02_argmax_agreement": 49,
  "near_ties": [
   {
    "id": "r18-slot",
    "oracle_margin": 0.009739205241203308,
    "oracle_argmax": 253,
    "option_argmax": 253,
    "p": [
     0.00015901612641755491,
     0.00015668613195884973,
     1.954942308657337e-05,
     3.6879966501146555e-05,
     1.601735311851371e-05,
     9.073647561308462e-06,
     2.3772492568241432e-05,
     6.854572347947396e-06,
     2.56846988122561e-06,
     4.846037336392328e-05,
     3.4505364965298213e-06,
     3.1118051992962137e-06,
     1.7958611351787113e-05,
     2.6259738206135808e-06,
     3.660404672700679e-06,
     9.345512808067724e-06,
     1.3316865761225927e-06,
     2.7856904125656e-06,
     1.1878885743499268e-05,
     1.2834397011829424e-06,
     2.8480574201239506e-06,
     1.1490847100503743e-05,
     1.1321013744236552e-06,
     1.3817469834975782e-06,
     5.026918756811938e-07,
     2.3160316686698934e-07,
     1.0515557278267806e-06,
     5.254527195575065e-07,
     1.3514893453248078e-06,
     1.0991680028382689e-06,
     2.4212079097196693e-06,
     6.852813072555364e-08,
     4.1799285099841654e-07,
     1.1489362350403098e-06,
     1.911887466121698e-06,
     3.4247241842422227e-07,
     4.2735103988889023e-07,
     3.3010708193614846e-06,
     2.2823887775302865e-06,
     1.326952133240411e-05,
     5.452054097077053e-07,
     5.826502729178173e-07,
     1.6251367185304844e-07,
     1.2646340792343835e-06,
     1.0991680028382689e-06,
     2.51190300559756e-07,
     1.601529788786138e-06,
     1.7115230548370164e-06,
     8.615670594736002e-07,
     8.181858106581785e-07,
     8.363963388546836e-08,
     3.7143598774491693e-07,
     1.5895477645244682e-07,
     5.41196413905709e-07,
     6.460738859459525e-07,
     2.9990651455591433e-06,
     2.684765149751911e-06,
     8.24352537165396e-06,
     3.252701617384446e-06,
     2.5311601348221302e-05,
     1.6490901089127874e-08,
     4.600256886533316e-08,
     1.4654111701872807e-08,
     5.741129029956937e-07,
     5.657006454384828e-07,
     7.004433144608413e-10,
     2.492792416930456e-09,
     4.029007413919317e-06,
     1.6733997521001243e-09,
     6.319260705822671e-07,
     4.7026458283028205e-09,
     1.4336893627842073e-06,
     2.0359322661533952e-05,
     2.3160316686698934e-07,
     3.476097617749474e-06,
     1.3611515470302038e-08,
     1.4657872497991775e-06,
     2.5115806323583456e-08,
     6.956502602406545e-06,
     2.2157088608310005e-07,
     5.294130914990092e-06,
     7.324405828512681e-07,
     2.587828566902317e-05,
     1.3714105762119289e-07,
     5.372857685870258e-06,
     1.566459218338423e-06,
     5.870417226105928e-06,
     1.6740439150453312e-06,
     2.7246890113019617e-06,
     2.166911095002888e-08,
     3.25228427300317e-07,
     8.180808208635426e-08,
     1.3415514104053727e-06,
     1.4336893627842073e-06,
     2.0431959910638398e-06,
     3.7143598774491693e-07,
     1.998454081331147e-06,
     2.475732435414102e-05,
     2.645087135988433e-07,
     1.7369741271977546e-06,
     5.138139691496235e-09,
     5.493146545632044e-06,
     4.8454157877131365e-06,
     7.433322366523498e-07,
     5.027563929616008e-06,
     6.271183572437167e-09,
     5.913145173508383e-07,
     1.6371754441024677e-07,
     1.8022701624431647e-06,
     2.6643395045766738e-08,
     1.0324641152692493e-05,
     2.703959545158341e-08,
     2.420586575624384e-08,
     8.00064547945567e-09,
     1.1574471727726632e-06,
     3.6205707147018984e-05,
     1.371586563436722e-06,
     1.1070273586710755e-08,
     1.0133279459978439e-07,
     1.3415514104053727e-06,
     8.744910701352637e-06,
     2.6069987143273465e-05,
     1.0324641152692493e-05,
     2.765206090771244e-06,
     2.0061006580363028e-05,
     1.4546357007816368e-08,
     9.137354872734704e-09,
     2.215993163190433e-06,
     4.567601536109578e-06,
     3.5005001119969847e-09,
     2.1994159737914742e-07,
     3.157674086651241e-07,
     4.498942551833807e-09,
     5.2920885451612776e-09,
     8.42808458401123e-06,
     1.0827845287053606e-08,
     8.24246797037631e-07,
     1.183360723189253e-06,
     4.368066353066524e-09,
     1.2832751394853403e-07,
     1.8978286107085296e-06,
     3.349729240653687e-07,
     1.5097067489477922e-06,
     1.6253452486125752e-06,
     3.325524176034378e-06,
     1.0361476370235323e-06,
     4.368066353066524e-09,
     2.420586575624384e-08,
     4.3691875362128485e-07,
     1.0671927839212003e-06,
     3.2763765034360404e-07,
     8.303525760311459e-07,
     7.217084316835098e-07,
     1.312173935730243e-06,
     6.957395089557394e-05,
     8.2445825682953e-05,
     0.00019053347932640463,
     0.000124183323350735,
     1.686661380517762e-05,
     3.940779606637079e-06,
     3.6205707147018984e-05,
     4.5012562623014674e-05,
     5.084181611891836e-05,
     5.084181611891836e-05,
     2.457212076478754e-06,
     0.0006380286649800837,
     2.4848864995874465e-05,
     2.167744969483465e-05,
     6.853692866570782e-07,
     8.55341295391554e-06,
     1.538017750135623e-05,
     1.5899571735644713e-05,
     0.00028593215392902493,
     3.158079152854043e-06,
     1.8091644960804842e-05,
     1.625553886697162e-05,
     6.366072398122924e-07,
     2.088939936584211e-06,
     4.4015533262609097e-07,
     1.4126820815363317e-06,
     6.413230835278227e-07,
     1.0360147939536546e-07,
     1.4443097597904853e-06,
     7.656039997527841e-07,
     3.882538806010416e-07,
     1.9260503449913813e-06,
     1.6373854805351584e-06,
     1.4126820815363317e-06,
     5.783658707514405e-07,
     3.2763765034360404e-07,
     4.305167351503769e-07,
     1.8156210899178404e-06,
     8.002690492503461e-07,
     2.549583086874918e-06,
     4.434159279753658e-07,
     1.0991680028382689e-06,
     4.2735103988889023e-07,
     2.911820729423198e-06,
     1.8978286107085296e-06,
     1.3514893453248078e-06,
     2.2324086330627324e-06,
     9.344313411929761e-07,
     7.600717708555749e-06,
     5.698913128071581e-07,
     2.013257926591905e-06,
     4.953897132509155e-06,
     1.3025251064391341e-06,
     4.774418357555987e-06,
     0.009822138585150242,
     7.769888838993211e-07,
     4.500674549490213e-06,
     1.8700208102018223e-06,
     2.7856904125656e-06,
     3.6875201203656616e-06,
     2.8691551960946526e-06,
     3.770077910303371e-06,
     1.1921267741854535e-06,
     6.273598046391271e-06,
     3.9993806240090635e-06,
     7.490308053093031e-05,
     2.4666143872309476e-05,
     0.018862541764974594,
     0.003608161583542824,
     0.0070496490225195885,
     0.08074595779180527,
     0.008825290948152542,
     0.022112583741545677,
     0.012658679857850075,
     0.009349115192890167,
     0.024744026362895966,
     0.04498836770653725,
     0.07970976829528809,
     0.0031130046118050814,
     0.017039315775036812,
     0.059278689324855804,
     0.03922849893569946,
     0.027545301243662834,
     0.01124873198568821,
     0.04414552077651024,
     0.02785193920135498,
     0.018754635006189346,
     0.005899829789996147,
     0.005434760823845863,
     0.05824866518378258,
     0.015111787244677544,
     0.09974077343940735,
     0.006386995781213045,
     0.022665081545710564,
     0.005242695566266775,
     0.01581607013940811,
     0.0049239033833146095,
     0.05686810240149498,
     0.015947936102747917,
     0.10817603766918182,
     0.006020799744874239
    ],
    "p_oracle": [
     0.0001469101116526872,
     0.0001461399078834802,
     1.8727199858403765e-05,
     3.445339825702831e-05,
     1.491959483246319e-05,
     8.525082193955313e-06,
     2.203234726039227e-05,
     6.327398750727298e-06,
     2.4345908968825825e-06,
     4.6558663598261774e-05,
     3.151977125526173e-06,
     2.988307187479222e-06,
     1.7149484847323038e-05,
     2.3426230200129794e-06,
     3.3076789804908913e-06,
     8.558708032069262e-06,
     1.175259626506886e-06,
     2.548649490563548e-06,
     1.0875735824811272e-05,
     1.174447334051365e-06,
     2.7018331820727326e-06,
     1.073516978067346e-05,
     1.0747947953859693e-06,
     1.2419112636052887e-06,
     4.439075667050929e-07,
     2.042345670361101e-07,
     9.404049592376396e-07,
     4.902366299575078e-07,
     1.2479507631724118e-06,
     9.923292054736521e-07,
     2.2280037228483707e-06,
     5.986074569364064e-08,
     3.658147988971905e-07,
     1.0361229669797467e-06,
     1.7353581824863795e-06,
     3.016077982920251e-07,
     3.907217376308836e-07,
     2.925652097474085e-06,
     1.966859144886257e-06,
     1.2392310054565314e-05,
     5.118952799421095e-07,
     5.359339070309943e-07,
     1.4427850203446724e-07,
     1.123934225688572e-06,
     1.0238458116873517e-06,
     2.1720221354826208e-07,
     1.4275401554186828e-06,
     1.5700845779065276e-06,
     8.077286679508688e-07,
     7.595568831675337e-07,
     7.496300469256312e-08,
     3.4219951317027153e-07,
     1.4025690120433865e-07,
     5.061549472884508e-07,
     5.872788619853964e-07,
     2.820014060489484e-06,
     2.5274971449107397e-06,
     7.469083357136697e-06,
     2.887191612899187e-06,
     2.2984444512985647e-05,
     1.4499901723752373e-08,
     4.224759564408487e-08,
     1.3127833931037003e-08,
     5.267225446914381e-07,
     5.032041485719674e-07,
     5.929003399351984e-10,
     2.150327471639457e-09,
     3.6878504943160806e-06,
     1.5137613385007853e-09,
     5.771068458670925e-07,
     4.05874622799729e-09,
     1.283262236029259e-06,
     1.8895223547588103e-05,
     2.1562215124504291e-07,
     3.2647783427819377e-06,
     1.216329259534632e-08,
     1.3845196917827707e-06,
     2.249413455501781e-08,
     6.356727226375369e-06,
     2.0436063152828865e-07,
     4.80034123029327e-06,
     6.884560548314766e-07,
     2.4657274479977787e-05,
     1.261227424720346e-07,
     5.058232545707142e-06,
     1.4855725112283835e-06,
     5.549032721319236e-06,
     1.5243693951561e-06,
     2.5501326490484644e-06,
     1.9503026749134733e-08,
     2.894741442105442e-07,
     7.430752901882443e-08,
     1.2425402928784024e-06,
     1.359587031402043e-06,
     1.943861207109876e-06,
     3.3811633670666197e-07,
     1.8900060467785806e-06,
     2.3787959435139783e-05,
     2.389311362094304e-07,
     1.6118586927404976e-06,
     4.5472798859691466e-09,
     5.195611720409943e-06,
     4.406738753459649e-06,
     6.553700018230302e-07,
     4.563604306895286e-06,
     5.59060486793328e-09,
     5.400765417107323e-07,
     1.514987673090218e-07,
     1.6380289480366628e-06,
     2.4197854386898143e-08,
     9.32275997911347e-06,
     2.393656828303392e-08,
     2.1264302318968475e-08,
     6.9980532479974045e-09,
     1.081376012734836e-06,
     3.306658618384972e-05,
     1.222553464685916e-06,
     9.822396229708374e-09,
     9.009193746578603e-08,
     1.2590995766004198e-06,
     7.929240382509306e-06,
     2.503086216165684e-05,
     9.561872502672486e-06,
     2.482203854015097e-06,
     1.9406297724344768e-05,
     1.3104779483796847e-08,
     8.068131940319745e-09,
     2.05597734748153e-06,
     4.0540408008382656e-06,
     3.169926543478141e-09,
     1.9746653379115742e-07,
     2.882631235934241e-07,
     3.9420209319018795e-09,
     4.785592810918615e-09,
     8.075558071141131e-06,
     9.4671372963262e-09,
     7.656602178940375e-07,
     1.0820309626069502e-06,
     3.729021536003074e-09,
     1.2206049859742052e-07,
     1.735182763695775e-06,
     3.0206345513761335e-07,
     1.4078510730541893e-06,
     1.526253186057147e-06,
     3.32214199261216e-06,
     9.722509730636375e-07,
     3.903593004395134e-09,
     2.2041644953674222e-08,
     4.0664875200491224e-07,
     9.705481716082431e-07,
     2.8843609811701754e-07,
     7.440521017088031e-07,
     6.660162625848898e-07,
     1.2110650686736335e-06,
     6.550806574523449e-05,
     7.979811925906688e-05,
     0.0001818382297642529,
     0.00011510254262248054,
     1.574801717652008e-05,
     3.7079730645928066e-06,
     3.386830576346256e-05,
     4.226343662594445e-05,
     4.857313251704909e-05,
     4.7070567234186456e-05,
     2.2569599877897417e-06,
     0.0006120369071140885,
     2.3661097657168284e-05,
     2.0005791157018393e-05,
     6.244418386813777e-07,
     7.96031963545829e-06,
     1.4662005014542956e-05,
     1.4956691302359104e-05,
     0.0002772198058664799,
     2.891156555051566e-06,
     1.6640678950352594e-05,
     1.5442494259332307e-05,
     5.737019250773301e-07,
     1.9168603557773167e-06,
     3.9845309629527037e-07,
     1.2928065871165018e-06,
     5.787465511275514e-07,
     9.430754488448656e-08,
     1.3109068959238357e-06,
     6.849967917332833e-07,
     3.59871819455293e-07,
     1.7630660522627295e-06,
     1.5343573522841325e-06,
     1.2713866226476966e-06,
     5.288820261739602e-07,
     2.9492520070562023e-07,
     3.9059952428033284e-07,
     1.6423742863480584e-06,
     7.348767780968046e-07,
     2.3556704036309384e-06,
     4.0035379811342864e-07,
     9.642487839300884e-07,
     3.915128559128789e-07,
     2.6290808818885125e-06,
     1.7516496200187248e-06,
     1.2177035841887118e-06,
     2.065924718408496e-06,
     8.670426154822053e-07,
     6.833991392340977e-06,
     4.97766279750067e-07,
     1.818960640775913e-06,
     4.63209062218084e-06,
     1.1824243983937777e-06,
     4.360892944532679e-06,
     0.009755837731063366,
     7.05798640865396e-07,
     4.1220273487851955e-06,
     1.7139896044682246e-06,
     2.5792653559619794e-06,
     3.3125340905826306e-06,
     2.6990644528268604e-06,
     3.3724088552844478e-06,
     1.0858207133424003e-06,
     5.779492767032934e-06,
     3.6232527236279566e-06,
     6.950746319489554e-05,
     2.359280188102275e-05,
     0.018962303176522255,
     0.0035667235497385263,
     0.006895596627146006,
     0.08383864909410477,
     0.008625222370028496,
     0.022189911454916,
     0.012730501592159271,
     0.009463458321988583,
     0.024959232658147812,
     0.0444105863571167,
     0.08021362125873566,
     0.0028592399321496487,
     0.017242614179849625,
     0.060589153319597244,
     0.039358872920274734,
     0.02689761482179165,
     0.011007904075086117,
     0.044373709708452225,
     0.02796688862144947,
     0.0190079715102911,
     0.005807948764413595,
     0.005553585942834616,
     0.05795270577073097,
     0.014917745254933834,
     0.0977199524641037,
     0.006410952657461166,
     0.022017937153577805,
     0.005312574561685324,
     0.015614868141710758,
     0.004747352562844753,
     0.057182542979717255,
     0.01578008383512497,
     0.107459157705307,
     0.005929228384047747
    ]
   }
  ],
  "max_abs_delta_p": 0.020812660455703735,
  "mean_of_row_mean_abs_delta_p": 0.0006594286450542834,
  "max_abs_delta_abstain": 0.01883678138256073,
  "max_abs_delta_raw_logits": 0.17230606079101562,
  "finite_all": true,
  "nonconstant_all": true,
  "reset_identical": true,
  "session_resets_identical": true,
  "provisional_fp16_max_delta_p_expectation": 0.02,
  "provisional_expectation_exceeded": null,
  "worst_five": [
   {
    "id": "r05-dry",
    "request_id": "r05",
    "question_id": "dry",
    "type": "noul",
    "qtype_index": 2,
    "kind": "slot",
    "tokens": 73,
    "nopts": 2,
    "temperature": 1.0067671537399292,
    "raw_logits": [
     -0.0214996337890625,
     0.187255859375,
     -1.2783203125,
     -2.208984375,
     -2.109375,
     -2.96484375,
     -2.615234375,
     -3.0078125,
     -4.6015625,
     -2.654296875,
     -1.9697265625,
     -3.34765625,
     -3.837890625,
     -2.427734375,
     -3.12109375,
     -3.947265625,
     -4.01953125,
     -5.33984375,
     -3.328125,
     -3.138671875,
     -3.525390625,
     -2.0703125,
     -5.10546875,
     -6.08984375,
     -3.443359375,
     -4.78125,
     -2.71875,
     -4.7734375,
     -3.650390625,
     -5.42578125,
     -4.03515625,
     -3.953125,
     -5.5,
     -4.03515625,
     -2.65234375,
     -4.69140625,
     -5.21875,
     -5.44921875,
     -4.5703125,
     -2.580078125,
     -6.33984375,
     -4.19921875,
     -4.59375,
     -5.5234375,
     -3.236328125,
     -4.51171875,
     -3.830078125,
     -4.52734375,
     -4.58984375,
     -4.1953125,
     -5.09765625,
     -4.453125,
     -3.0390625,
     -4.7421875,
     -2.7578125,
     -3.44921875,
     -3.6328125,
     -4.40625,
     -4.0625,
     -5.5,
     -7.20703125,
     -6.375,
     -7.8203125,
     -5.83984375,
     -3.287109375,
     -7.1875,
     -7.1953125,
     -2.28515625,
     -7.4921875,
     -4.23828125,
     -7.53515625,
     -4.84765625,
     -2.533203125,
     -5.3671875,
     -3.732421875,
     -5.59375,
     -4.29296875,
     -6.265625,
     -3.603515625,
     -5.44140625,
     -4.13671875,
     -3.498046875,
     -2.388671875,
     -4.04296875,
     -3.29296875,
     -3.94921875,
     -2.470703125,
     -3.392578125,
     -3.068359375,
     -4.1640625,
     -3.150390625,
     -3.724609375,
     -3.38671875,
     -3.767578125,
     -4.359375,
     -4.296875,
     -4.5625,
     -3.35546875,
     -2.64453125,
     -3.87109375,
     -5.171875,
     -4.88671875,
     -3.07421875,
     -3.90625,
     -4.0703125,
     -5.4921875,
     -3.10546875,
     -4.7109375,
     -3.849609375,
     -5.3125,
     -4.14453125,
     -4.5390625,
     -4.39453125,
     -5.62109375,
     -4.2265625,
     -3.171875,
     -4.54296875,
     -5.13671875,
     -3.876953125,
     -3.32421875,
     -3.2109375,
     -4.0703125,
     -2.775390625,
     -3.3828125,
     -2.376953125,
     -4.84765625,
     -5.03125,
     -2.642578125,
     -2.59375,
     -4.71484375,
     -5.17578125,
     -2.607421875,
     -5.0859375,
     -5.875,
     -5.19921875,
     -4.3828125,
     -3.408203125,
     -4.17578125,
     -5.46875,
     -2.291015625,
     -3.33984375,
     -3.943359375,
     -3.009765625,
     -2.970703125,
     -4.328125,
     -4.7265625,
     -5.55078125,
     -4.96875,
     -4.796875,
     -3.8359375,
     -4.125,
     -2.326171875,
     -0.81689453125,
     -2.58203125,
     -1.95703125,
     -0.95703125,
     -1.3876953125,
     -1.560546875,
     -2.736328125,
     -0.75537109375,
     -1.76171875,
     -1.6494140625,
     -0.85107421875,
     -1.1318359375,
     -2.533203125,
     -1.9521484375,
     -2.3671875,
     -1.658203125,
     -3.482421875,
     -1.865234375,
     -1.732421875,
     -2.099609375,
     -1.751953125,
     -2.5859375,
     -0.72265625,
     -1.6171875,
     -3.1796875,
     -3.03125,
     -2.583984375,
     -2.625,
     -3.11328125,
     -2.173828125,
     -2.357421875,
     -1.8974609375,
     -2.068359375,
     -1.6806640625,
     -1.400390625,
     -1.392578125,
     -1.9912109375,
     -1.9560546875,
     -0.493896484375,
     -1.76953125,
     -2.31640625,
     -1.2099609375,
     -1.2705078125,
     -1.58203125,
     -1.87109375,
     -1.37109375,
     -1.6484375,
     -0.96630859375,
     -1.0517578125,
     -1.6474609375,
     -1.205078125,
     -1.2919921875,
     -1.9794921875,
     -0.85400390625,
     -2.2578125,
     -2.458984375,
     -0.1263427734375,
     -1.681640625,
     -0.64990234375,
     -1.03125,
     -0.1162109375,
     -0.87255859375,
     -2.4765625,
     -0.7900390625,
     -1.251953125,
     -2.1328125,
     -1.3466796875,
     -1.6337890625,
     -1.513671875,
     -1.6259765625,
     -1.375,
     -0.80517578125,
     -0.0225067138671875,
     0.364990234375,
     -0.4169921875,
     0.9052734375,
     0.91943359375,
     -0.268310546875,
     -0.30615234375,
     -0.3427734375,
     -0.08544921875,
     -0.94384765625,
     0.267333984375,
     1.0244140625,
     1.388671875,
     -0.346435546875,
     -0.22021484375,
     -0.64990234375,
     0.609375,
     0.5751953125,
     -0.459716796875,
     0.166748046875,
     0.07275390625,
     0.4287109375,
     0.56640625,
     -0.54833984375,
     -0.63427734375,
     1.9140625,
     -0.364013671875,
     -0.57373046875,
     -0.333740234375,
     0.361328125,
     -0.712890625,
     -7.9453125
    ],
    "raw256_argmax": 249,
    "option_argmax": 1,
    "oracle_argmax": 1,
    "argmax_agrees": true,
    "p_full": [
     0.44827014207839966,
     0.5515587329864502,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.00017116019444074482
    ],
    "p": [
     0.4483468532562256,
     0.5516531467437744
    ],
    "p_oracle": [
     0.4691595137119293,
     0.5308405160903931
    ],
    "abstain": 0.00017116019444074482,
    "abstain_oracle": 0.00016973857418633997,
    "abs_delta_abstain": 1.4216202544048429e-06,
    "oracle_margin": 0.061681002378463745,
    "max_abs_delta_p": 0.020812660455703735,
    "mean_abs_delta_p": 0.02081264555454254,
    "max_abs_delta_raw_logits": 0.07616090774536133,
    "mean_abs_delta_raw_logits": 0.007669182610698044,
    "logits_min": -7.9453125,
    "logits_max": 1.9140625,
    "raw_logits_min": -7.9453125,
    "raw_logits_max": 1.9140625,
    "raw_logits_absmax": 7.9453125,
    "oracle_logits_min": -7.923232078552246,
    "oracle_logits_max": 1.9354100227355957,
    "oracle_raw_logits_absmax": 7.923232078552246,
    "finite": true,
    "nonconstant": true,
    "wall_seconds_contended": 0.4102001660503447,
    "seconds": 0.4102001660503447,
    "full_vocab_argmax_id": 249,
    "full_vocab_argmax_is_label": false
   },
   {
    "id": "r14-fill",
    "request_id": "r14",
    "question_id": "fill",
    "type": "score",
    "qtype_index": 1,
    "kind": "slot",
    "tokens": 87,
    "nopts": 3,
    "temperature": 1.0431410074234009,
    "raw_logits": [
     2.587890625,
     1.7802734375,
     -2.447265625,
     -2.8515625,
     -2.482421875,
     -3.21875,
     -3.0078125,
     -3.83203125,
     -4.07421875,
     -2.71875,
     -2.68359375,
     -3.78125,
     -4.5078125,
     -2.84765625,
     -3.5703125,
     -4.08984375,
     -3.8359375,
     -4.84375,
     -2.73828125,
     -3.373046875,
     -3.19921875,
     -1.6328125,
     -4.94140625,
     -6.13671875,
     -3.1875,
     -4.50390625,
     -2.16796875,
     -4.93359375,
     -4.03515625,
     -6.49609375,
     -3.9296875,
     -4.53515625,
     -5.6484375,
     -4.61328125,
     -3.060546875,
     -4.921875,
     -5.0,
     -5.66796875,
     -4.140625,
     -2.947265625,
     -6.69140625,
     -4.43359375,
     -4.9375,
     -5.53125,
     -3.7578125,
     -4.671875,
     -3.810546875,
     -4.421875,
     -4.95703125,
     -4.41015625,
     -5.359375,
     -4.38671875,
     -3.005859375,
     -4.83203125,
     -3.021484375,
     -3.44921875,
     -3.841796875,
     -4.3828125,
     -4.359375,
     -5.81640625,
     -7.26171875,
     -6.5078125,
     -7.8828125,
     -5.71875,
     -3.6484375,
     -6.98046875,
     -7.2734375,
     -2.19140625,
     -7.34765625,
     -4.05859375,
     -7.65234375,
     -4.80078125,
     -2.78515625,
     -5.4140625,
     -3.669921875,
     -5.8203125,
     -4.125,
     -6.56640625,
     -3.609375,
     -5.4375,
     -4.08203125,
     -3.69140625,
     -2.52734375,
     -4.23828125,
     -3.2578125,
     -3.8984375,
     -2.369140625,
     -3.681640625,
     -2.91796875,
     -4.53125,
     -3.296875,
     -3.92578125,
     -3.736328125,
     -3.6640625,
     -4.06640625,
     -4.05859375,
     -4.796875,
     -3.54296875,
     -2.998046875,
     -4.00390625,
     -5.4375,
     -4.7578125,
     -3.171875,
     -3.90625,
     -4.203125,
     -5.78125,
     -3.7890625,
     -5.16796875,
     -3.9375,
     -5.578125,
     -4.265625,
     -4.703125,
     -4.7734375,
     -6.12109375,
     -4.734375,
     -3.779296875,
     -4.62109375,
     -5.4921875,
     -3.884765625,
     -3.484375,
     -3.365234375,
     -4.04296875,
     -3.1328125,
     -3.537109375,
     -2.421875,
     -5.20703125,
     -5.4140625,
     -2.681640625,
     -2.650390625,
     -5.1171875,
     -5.40234375,
     -3.03515625,
     -5.27734375,
     -5.85546875,
     -4.96484375,
     -4.1640625,
     -3.5390625,
     -4.88671875,
     -5.6328125,
     -2.849609375,
     -3.541015625,
     -4.08984375,
     -3.16015625,
     -2.685546875,
     -4.80078125,
     -5.12890625,
     -5.78125,
     -4.9921875,
     -4.7421875,
     -4.21484375,
     -4.32421875,
     -2.732421875,
     -1.0185546875,
     -3.083984375,
     -2.603515625,
     -1.369140625,
     -1.794921875,
     -1.9111328125,
     -2.978515625,
     -0.724609375,
     -1.830078125,
     -2.001953125,
     -1.544921875,
     -1.1162109375,
     -2.578125,
     -1.939453125,
     -2.67578125,
     -2.1328125,
     -3.4609375,
     -2.02734375,
     -2.10546875,
     -2.490234375,
     -2.171875,
     -2.9296875,
     -0.79150390625,
     -2.10546875,
     -3.580078125,
     -3.380859375,
     -2.912109375,
     -2.5859375,
     -3.234375,
     -2.65625,
     -2.494140625,
     -2.14453125,
     -2.193359375,
     -1.7392578125,
     -1.37890625,
     -1.4892578125,
     -2.36328125,
     -2.259765625,
     -0.55810546875,
     -1.630859375,
     -2.59765625,
     -1.3681640625,
     -1.455078125,
     -1.5234375,
     -2.0625,
     -1.5927734375,
     -1.84765625,
     -1.0029296875,
     -0.96728515625,
     -1.7578125,
     -1.39453125,
     -1.55859375,
     -2.265625,
     -0.9453125,
     -2.470703125,
     -2.638671875,
     -0.37890625,
     -2.23828125,
     -1.1455078125,
     -1.2490234375,
     -0.21435546875,
     -1.001953125,
     -2.470703125,
     -1.060546875,
     -1.623046875,
     -2.15625,
     -1.2197265625,
     -1.9501953125,
     -1.361328125,
     -1.7919921875,
     -1.0888671875,
     -0.912109375,
     -0.25732421875,
     0.0987548828125,
     -0.5361328125,
     1.103515625,
     0.8486328125,
     -0.2958984375,
     -0.471923828125,
     -0.74853515625,
     0.08428955078125,
     -0.900390625,
     0.11810302734375,
     0.90234375,
     1.3935546875,
     -0.311767578125,
     0.2464599609375,
     -0.8466796875,
     0.355712890625,
     0.59033203125,
     -0.54736328125,
     -0.148193359375,
     -0.0977783203125,
     0.46728515625,
     0.4794921875,
     -0.2437744140625,
     -0.90185546875,
     2.015625,
     -0.361328125,
     -0.4443359375,
     -0.204345703125,
     0.3369140625,
     -0.57763671875,
     -5.2890625
    ],
    "raw256_argmax": 0,
    "option_argmax": 0,
    "oracle_argmax": 0,
    "argmax_agrees": true,
    "p_full": [
     0.6804565787315369,
     0.31373459100723267,
     0.005451255012303591,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0003575703885871917
    ],
    "p": [
     0.6807000041007996,
     0.3138468265533447,
     0.005453204736113548
    ],
    "p_oracle": [
     0.675521969795227,
     0.3190520703792572,
     0.005425968673080206
    ],
    "abstain": 0.0003575703885871917,
    "abstain_oracle": 0.000354081392288208,
    "abs_delta_abstain": 3.488996298983693e-06,
    "oracle_margin": 0.35646989941596985,
    "max_abs_delta_p": 0.005205243825912476,
    "mean_abs_delta_p": 0.0034701713981727758,
    "max_abs_delta_raw_logits": 0.026024818420410156,
    "mean_abs_delta_raw_logits": 0.004977937787771225,
    "logits_min": -7.8828125,
    "logits_max": 2.587890625,
    "raw_logits_min": -7.8828125,
    "raw_logits_max": 2.587890625,
    "raw_logits_absmax": 7.8828125,
    "oracle_logits_min": -7.882862567901611,
    "oracle_logits_max": 2.586092710494995,
    "oracle_raw_logits_absmax": 7.882862567901611,
    "finite": true,
    "nonconstant": true,
    "wall_seconds_contended": 0.4891923750983551,
    "seconds": 0.4891923750983551,
    "full_vocab_argmax_id": 0,
    "full_vocab_argmax_is_label": true
   },
   {
    "id": "r05-task",
    "request_id": "r05",
    "question_id": "task",
    "type": "choice",
    "qtype_index": 0,
    "kind": "slot",
    "tokens": 99,
    "nopts": 3,
    "temperature": 1.058533787727356,
    "raw_logits": [
     1.09375,
     -1.5341796875,
     -4.53515625,
     -3.365234375,
     -3.244140625,
     -3.193359375,
     -3.009765625,
     -4.37109375,
     -5.0703125,
     -3.564453125,
     -2.951171875,
     -3.109375,
     -5.2734375,
     -3.4140625,
     -3.943359375,
     -3.90234375,
     -4.28515625,
     -4.734375,
     -3.44140625,
     -3.88671875,
     -3.595703125,
     -2.474609375,
     -5.42578125,
     -5.8671875,
     -4.26953125,
     -5.4765625,
     -2.953125,
     -5.6328125,
     -4.50390625,
     -6.37890625,
     -4.66796875,
     -4.39453125,
     -6.57421875,
     -4.40234375,
     -3.8046875,
     -5.38671875,
     -5.75,
     -5.7421875,
     -4.34375,
     -3.126953125,
     -6.1875,
     -5.2734375,
     -5.5,
     -5.51171875,
     -3.91796875,
     -5.1171875,
     -4.34765625,
     -4.96875,
     -5.41796875,
     -4.6640625,
     -5.53515625,
     -5.046875,
     -3.8203125,
     -5.22265625,
     -3.3359375,
     -3.728515625,
     -4.0234375,
     -4.1640625,
     -4.6796875,
     -5.546875,
     -7.62109375,
     -7.11328125,
     -8.2578125,
     -6.30859375,
     -3.8828125,
     -7.5859375,
     -7.96875,
     -2.86328125,
     -8.1171875,
     -4.19921875,
     -8.0625,
     -5.46484375,
     -2.71875,
     -5.6015625,
     -4.07421875,
     -6.3671875,
     -4.27734375,
     -7.16796875,
     -4.01171875,
     -6.0859375,
     -4.27734375,
     -4.1953125,
     -2.939453125,
     -4.76171875,
     -3.716796875,
     -4.61328125,
     -2.6796875,
     -4.06640625,
     -3.51171875,
     -4.98046875,
     -3.966796875,
     -4.68359375,
     -4.515625,
     -3.86328125,
     -4.6171875,
     -4.046875,
     -5.0625,
     -3.556640625,
     -2.998046875,
     -4.41796875,
     -6.29296875,
     -4.875,
     -3.544921875,
     -4.76171875,
     -5.08984375,
     -6.2421875,
     -4.1328125,
     -5.55078125,
     -4.05859375,
     -5.96875,
     -4.23828125,
     -5.05859375,
     -5.3046875,
     -6.3046875,
     -4.765625,
     -3.58984375,
     -4.703125,
     -6.15625,
     -4.734375,
     -4.01953125,
     -3.14453125,
     -4.12890625,
     -3.0078125,
     -3.931640625,
     -2.216796875,
     -5.44140625,
     -5.9765625,
     -2.732421875,
     -3.103515625,
     -5.96875,
     -5.6953125,
     -3.791015625,
     -6.1015625,
     -6.78125,
     -5.2109375,
     -5.1015625,
     -3.783203125,
     -4.609375,
     -6.4296875,
     -2.939453125,
     -3.98828125,
     -4.43359375,
     -3.302734375,
     -3.103515625,
     -4.328125,
     -5.26953125,
     -6.23828125,
     -5.921875,
     -4.9609375,
     -4.32421875,
     -4.5078125,
     -2.880859375,
     -1.3369140625,
     -3.28515625,
     -2.08203125,
     -1.8193359375,
     -1.4052734375,
     -1.6337890625,
     -2.962890625,
     -1.166015625,
     -2.146484375,
     -1.6953125,
     -1.607421875,
     -1.3408203125,
     -2.384765625,
     -2.2421875,
     -2.984375,
     -2.529296875,
     -3.6953125,
     -2.408203125,
     -2.5234375,
     -2.681640625,
     -2.392578125,
     -3.19921875,
     -1.1767578125,
     -1.9677734375,
     -3.93359375,
     -3.623046875,
     -3.65625,
     -3.13671875,
     -4.6328125,
     -3.15625,
     -3.126953125,
     -3.126953125,
     -3.22265625,
     -2.544921875,
     -1.9423828125,
     -2.23046875,
     -2.8203125,
     -2.705078125,
     -1.6455078125,
     -2.26171875,
     -3.390625,
     -2.044921875,
     -1.8759765625,
     -1.9423828125,
     -2.59375,
     -1.8896484375,
     -2.294921875,
     -1.4296875,
     -1.8056640625,
     -2.453125,
     -1.265625,
     -2.373046875,
     -2.4296875,
     -1.9560546875,
     -2.43359375,
     -2.576171875,
     -0.18212890625,
     -2.650390625,
     -1.7978515625,
     -1.595703125,
     -0.7685546875,
     -1.2861328125,
     -3.171875,
     -1.4716796875,
     -2.46484375,
     -2.369140625,
     -1.8134765625,
     -2.546875,
     -1.791015625,
     -1.240234375,
     -1.2470703125,
     -0.64453125,
     -0.11114501953125,
     -0.2138671875,
     -0.97314453125,
     0.59619140625,
     0.429931640625,
     -0.29248046875,
     -0.2459716796875,
     -0.53759765625,
     -0.02301025390625,
     -0.84716796875,
     -0.2169189453125,
     1.279296875,
     0.99658203125,
     -0.26611328125,
     -0.26904296875,
     -0.70361328125,
     0.37744140625,
     0.6796875,
     -0.496337890625,
     0.05926513671875,
     -0.1573486328125,
     0.491455078125,
     0.7080078125,
     -0.343017578125,
     0.023406982421875,
     1.8876953125,
     -0.6787109375,
     -0.2127685546875,
     -0.50244140625,
     -0.024169921875,
     -0.336669921875,
     -0.61376953125
    ],
    "raw256_argmax": 249,
    "option_argmax": 0,
    "oracle_argmax": 0,
    "argmax_agrees": true,
    "p_full": [
     0.7765794992446899,
     0.06486357003450394,
     0.0038085724227130413,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.15474838018417358
    ],
    "p": [
     0.9187554121017456,
     0.07673876732587814,
     0.004505844786763191
    ],
    "p_oracle": [
     0.9156041741371155,
     0.07993196696043015,
     0.004463807214051485
    ],
    "abstain": 0.15474838018417358,
    "abstain_oracle": 0.13591159880161285,
    "abs_delta_abstain": 0.01883678138256073,
    "oracle_margin": 0.8356721997261047,
    "max_abs_delta_p": 0.003193199634552002,
    "mean_abs_delta_p": 0.002128825057297945,
    "max_abs_delta_raw_logits": 0.12987357378005981,
    "mean_abs_delta_raw_logits": 0.021902231208514422,
    "logits_min": -8.2578125,
    "logits_max": 1.8876953125,
    "raw_logits_min": -8.2578125,
    "raw_logits_max": 1.8876953125,
    "raw_logits_absmax": 8.2578125,
    "oracle_logits_min": -8.236438751220703,
    "oracle_logits_max": 1.9041804075241089,
    "oracle_raw_logits_absmax": 8.236438751220703,
    "finite": true,
    "nonconstant": true,
    "wall_seconds_contended": 0.5561975000891834,
    "seconds": 0.5561975000891834,
    "full_vocab_argmax_id": 249,
    "full_vocab_argmax_is_label": false
   },
   {
    "id": "r18-slot",
    "request_id": "r18",
    "question_id": "slot",
    "type": "choice",
    "qtype_index": 0,
    "kind": "slot",
    "tokens": 1449,
    "nopts": 255,
    "temperature": 1.058533787727356,
    "raw_logits": [
     -5.0859375,
     -5.1015625,
     -7.3046875,
     -6.6328125,
     -7.515625,
     -8.1171875,
     -7.09765625,
     -8.4140625,
     -9.453125,
     -6.34375,
     -9.140625,
     -9.25,
     -7.39453125,
     -9.4296875,
     -9.078125,
     -8.0859375,
     -10.1484375,
     -9.3671875,
     -7.83203125,
     -10.1875,
     -9.34375,
     -7.8671875,
     -10.3203125,
     -10.109375,
     -11.1796875,
     -12.0,
     -10.3984375,
     -11.1328125,
     -10.1328125,
     -10.3515625,
     -9.515625,
     -13.2890625,
     -11.375,
     -10.3046875,
     -9.765625,
     -11.5859375,
     -11.3515625,
     -9.1875,
     -9.578125,
     -7.71484375,
     -11.09375,
     -11.0234375,
     -12.375,
     -10.203125,
     -10.3515625,
     -11.9140625,
     -9.953125,
     -9.8828125,
     -10.609375,
     -10.6640625,
     -13.078125,
     -11.5,
     -12.3984375,
     -11.1015625,
     -10.9140625,
     -9.2890625,
     -9.40625,
     -8.21875,
     -9.203125,
     -7.03125,
     -14.796875,
     -13.7109375,
     -14.921875,
     -11.0390625,
     -11.0546875,
     -18.140625,
     -16.796875,
     -8.9765625,
     -17.21875,
     -10.9375,
     -16.125,
     -10.0703125,
     -7.26171875,
     -12.0,
     -9.1328125,
     -15.0,
     -10.046875,
     -14.3515625,
     -8.3984375,
     -12.046875,
     -8.6875,
     -10.78125,
     -7.0078125,
     -12.5546875,
     -8.671875,
     -9.9765625,
     -8.578125,
     -9.90625,
     -9.390625,
     -14.5078125,
     -11.640625,
     -13.1015625,
     -10.140625,
     -10.0703125,
     -9.6953125,
     -11.5,
     -9.71875,
     -7.0546875,
     -11.859375,
     -9.8671875,
     -16.03125,
     -8.6484375,
     -8.78125,
     -10.765625,
     -8.7421875,
     -15.8203125,
     -11.0078125,
     -12.3671875,
     -9.828125,
     -14.2890625,
     -7.98046875,
     -14.2734375,
     -14.390625,
     -15.5625,
     -10.296875,
     -6.65234375,
     -10.1171875,
     -15.21875,
     -12.875,
     -10.140625,
     -8.15625,
     -7.0,
     -7.98046875,
     -9.375,
     -7.27734375,
     -14.9296875,
     -15.421875,
     -9.609375,
     -8.84375,
     -16.4375,
     -12.0546875,
     -11.671875,
     -16.171875,
     -16.0,
     -8.1953125,
     -15.2421875,
     -10.65625,
     -10.2734375,
     -16.203125,
     -12.625,
     -9.7734375,
     -11.609375,
     -10.015625,
     -9.9375,
     -9.1796875,
     -10.4140625,
     -16.203125,
     -14.390625,
     -11.328125,
     -10.3828125,
     -11.6328125,
     -10.6484375,
     -10.796875,
     -10.1640625,
     -5.9609375,
     -5.78125,
     -4.89453125,
     -5.34765625,
     -7.4609375,
     -9.0,
     -6.65234375,
     -6.421875,
     -6.29296875,
     -6.29296875,
     -9.5,
     -3.615234375,
     -7.05078125,
     -7.1953125,
     -10.8515625,
     -8.1796875,
     -7.55859375,
     -7.5234375,
     -4.46484375,
     -9.234375,
     -7.38671875,
     -7.5,
     -10.9296875,
     -9.671875,
     -11.3203125,
     -10.0859375,
     -10.921875,
     -12.8515625,
     -10.0625,
     -10.734375,
     -11.453125,
     -9.7578125,
     -9.9296875,
     -10.0859375,
     -11.03125,
     -11.6328125,
     -11.34375,
     -9.8203125,
     -10.6875,
     -9.4609375,
     -11.3125,
     -10.3515625,
     -11.3515625,
     -9.3203125,
     -9.7734375,
     -10.1328125,
     -9.6015625,
     -10.5234375,
     -8.3046875,
     -11.046875,
     -9.7109375,
     -8.7578125,
     -10.171875,
     -8.796875,
     -0.72119140625,
     -10.71875,
     -8.859375,
     -9.7890625,
     -9.3671875,
     -9.0703125,
     -9.3359375,
     -9.046875,
     -10.265625,
     -8.5078125,
     -8.984375,
     -5.8828125,
     -7.05859375,
     -0.03045654296875,
     -1.78125,
     -1.072265625,
     1.5087890625,
     -0.83447265625,
     0.1378173828125,
     -0.45263671875,
     -0.7734375,
     0.2568359375,
     0.8896484375,
     1.4951171875,
     -1.9375,
     -0.1380615234375,
     1.181640625,
     0.74462890625,
     0.370361328125,
     -0.57763671875,
     0.86962890625,
     0.382080078125,
     -0.036529541015625,
     -1.2607421875,
     -1.34765625,
     1.1630859375,
     -0.26513671875,
     1.732421875,
     -1.1767578125,
     0.1639404296875,
     -1.3857421875,
     -0.2169189453125,
     -1.4521484375,
     1.1376953125,
     -0.2081298828125,
     1.818359375,
     -1.2392578125,
     -4.66796875
    ],
    "raw256_argmax": 253,
    "option_argmax": 253,
    "oracle_argmax": 253,
    "argmax_agrees": true,
    "p_full": [
     0.00015897859702818096,
     0.00015664915554225445,
     1.9544810129445978e-05,
     3.687126445583999e-05,
     1.6013573258533143e-05,
     9.071506610780489e-06,
     2.3766882804920897e-05,
     6.852954811620293e-06,
     2.5678637030068785e-06,
     4.844893919653259e-05,
     3.449722271398059e-06,
     3.111070782324532e-06,
     1.795437310647685e-05,
     2.625354227347998e-06,
     3.65954088010767e-06,
     9.343307283415925e-06,
     1.3313723457031301e-06,
     2.7850330752698937e-06,
     1.1876082680828404e-05,
     1.283136839447252e-06,
     2.8473853035393404e-06,
     1.1488135896797758e-05,
     1.1318342103550094e-06,
     1.3814209296469926e-06,
     5.025732434660313e-07,
     2.3154851191975467e-07,
     1.0513075494600344e-06,
     5.253287440609711e-07,
     1.3511704537449987e-06,
     1.0989086831614259e-06,
     2.4206365196732804e-06,
     6.851195877288774e-08,
     4.1789422766669304e-07,
     1.1486650919323438e-06,
     1.9114363567496184e-06,
     3.4239161550431163e-07,
     4.2725019966383115e-07,
     3.300291837149416e-06,
     2.2818501292931614e-06,
     1.3266389942145906e-05,
     5.450767730508232e-07,
     5.82512768687593e-07,
     1.624753167561721e-07,
     1.2643356512853643e-06,
     1.0989086831614259e-06,
     2.511310128738842e-07,
     1.6011518937375513e-06,
     1.7111191255025915e-06,
     8.613637305643351e-07,
     8.179927135643084e-07,
     8.361989500826894e-08,
     3.7134833519303356e-07,
     1.589172740068534e-07,
     5.410686867435288e-07,
     6.459214318965678e-07,
     2.9983573313074885e-06,
     2.684131686692126e-06,
     8.241579962486867e-06,
     3.2519340038561495e-06,
     2.5305627787020057e-05,
     1.6487009091292748e-08,
     4.5991711772330746e-08,
     1.4650654023284915e-08,
     5.739774451285484e-07,
     5.655671770909976e-07,
     7.002780022524746e-10,
     2.4922042207720096e-09,
     4.028056537208613e-06,
     1.673004845770265e-09,
     6.317769702945952e-07,
     4.701536049367405e-09,
     1.4333510307551478e-06,
     2.0354518710519187e-05,
     2.3154851191975467e-07,
     3.475277253528475e-06,
     1.3608303817136402e-08,
     1.4654414144388284e-06,
     2.5109880397167217e-08,
     6.9548609644698445e-06,
     2.2151860434860282e-07,
     5.292881724017207e-06,
     7.322677220145124e-07,
     2.5872179321595468e-05,
     1.3710869950500637e-07,
     5.371589850255987e-06,
     1.56608962242899e-06,
     5.869032065675128e-06,
     1.6736488532842486e-06,
     2.7240459985478083e-06,
     2.1663996818688247e-08,
     3.251516886848549e-07,
     8.178877664022366e-08,
     1.341234792562318e-06,
     1.4333510307551478e-06,
     2.0427139588719e-06,
     3.7134833519303356e-07,
     1.9979825083282776e-06,
     2.475148176017683e-05,
     2.6444629952493415e-07,
     1.7365642861477681e-06,
     5.1369273279533445e-09,
     5.4918505156820174e-06,
     4.844272552873008e-06,
     7.431568178617454e-07,
     5.026377493777545e-06,
     6.269703867189946e-09,
     5.91174966757535e-07,
     1.6367890509627614e-07,
     1.8018448599832482e-06,
     2.66371085189121e-08,
     1.0322204616386443e-05,
     2.7033214777816283e-08,
     2.4200152992648327e-08,
     7.998757212135388e-09,
     1.1571740969884559e-06,
     3.619716517277993e-05,
     1.3712628970097285e-06,
     1.1067661453978417e-08,
     1.0130888483672607e-07,
     1.341234792562318e-06,
     8.742847057874314e-06,
     2.6063835321110673e-05,
     1.0322204616386443e-05,
     2.764553528322722e-06,
     2.0056273569935e-05,
     1.4542924198224227e-08,
     9.135198375531672e-09,
     2.2154702037369134e-06,
     4.566523784887977e-06,
     3.4996741060666636e-09,
     2.198896993377275e-07,
     3.156928869429976e-07,
     4.497880734533055e-09,
     5.290839766303179e-09,
     8.426095519098453e-06,
     1.0825289997740128e-08,
     8.240522788582894e-07,
     1.1830815083158086e-06,
     4.367035622010462e-09,
     1.2829723061713594e-07,
     1.8973807982547441e-06,
     3.3489388329144276e-07,
     1.5093504543983727e-06,
     1.6249616692221025e-06,
     3.3247395094804233e-06,
     1.035903096635593e-06,
     4.367035622010462e-09,
     2.4200152992648327e-08,
     4.3681563965947134e-07,
     1.066940967575647e-06,
     3.2756034329395334e-07,
     8.301566367663327e-07,
     7.215381288006029e-07,
     1.3118642527842894e-06,
     6.955753633519635e-05,
     8.242636977229267e-05,
     0.00019048851390834898,
     0.00012415401579346508,
     1.6862633856362663e-05,
     3.9398496483045164e-06,
     3.619716517277993e-05,
     4.5001939724897966e-05,
     5.082981806481257e-05,
     5.082981806481257e-05,
     2.456632273606374e-06,
     0.0006378780817613006,
     2.4843000574037433e-05,
     2.167233469663188e-05,
     6.852075671304192e-07,
     8.551394785172306e-06,
     1.5376548617496155e-05,
     1.5895819160505198e-05,
     0.0002858646912500262,
     3.15733382194594e-06,
     1.808737579267472e-05,
     1.625170261831954e-05,
     6.364570026562433e-07,
     2.08844699045585e-06,
     4.4005147969983227e-07,
     1.412348751728132e-06,
     6.411717663468153e-07,
     1.0357702961982795e-07,
     1.4439689266509959e-06,
     7.654233513676445e-07,
     3.88162249009838e-07,
     1.9255958250141703e-06,
     1.6369990589737426e-06,
     1.412348751728132e-06,
     5.782293897027557e-07,
     3.2756034329395334e-07,
     4.3041515596087265e-07,
     1.8151926042264677e-06,
     8.000802154128905e-07,
     2.5489814561296953e-06,
     4.4331127924124303e-07,
     1.0989086831614259e-06,
     4.2725019966383115e-07,
     2.9111336061760085e-06,
     1.8973807982547441e-06,
     1.3511704537449987e-06,
     2.2318818082567304e-06,
     9.34210845571215e-07,
     7.5989241850038525e-06,
     5.697568212781334e-07,
     2.012782942983904e-06,
     4.952727977070026e-06,
     1.3022176972299349e-06,
     4.773291493620491e-06,
     0.009819820523262024,
     7.768055070300761e-07,
     4.4996127144258935e-06,
     1.869579477897787e-06,
     2.7850330752698937e-06,
     3.6866499613097403e-06,
     2.8684780772891827e-06,
     3.7691881971113617e-06,
     1.1918455129489303e-06,
     6.272117843764136e-06,
     3.998437023255974e-06,
     7.488540722988546e-05,
     2.466032310621813e-05,
     0.018858090043067932,
     0.0036073101218789816,
     0.007047985680401325,
     0.08072690665721893,
     0.008823208510875702,
     0.02210736647248268,
     0.012655693106353283,
     0.009346908889710903,
     0.024738186970353127,
     0.04497775062918663,
     0.07969095557928085,
     0.0031122700311243534,
     0.017035294324159622,
     0.05926470085978508,
     0.039219241589307785,
     0.027538800612092018,
     0.011246077716350555,
     0.04413510486483574,
     0.027845365926623344,
     0.018750209361314774,
     0.005898437462747097,
     0.005433478392660618,
     0.05823491886258125,
     0.015108221210539341,
     0.09971723705530167,
     0.00638548843562603,
     0.02265973389148712,
     0.0052414583042263985,
     0.015812337398529053,
     0.004922741558402777,
     0.05685468390583992,
     0.01594417355954647,
     0.10815051198005676,
     0.006019379012286663,
     0.00023595143284182996
    ],
    "p": [
     0.00015901612641755491,
     0.00015668613195884973,
     1.954942308657337e-05,
     3.6879966501146555e-05,
     1.601735311851371e-05,
     9.073647561308462e-06,
     2.3772492568241432e-05,
     6.854572347947396e-06,
     2.56846988122561e-06,
     4.846037336392328e-05,
     3.4505364965298213e-06,
     3.1118051992962137e-06,
     1.7958611351787113e-05,
     2.6259738206135808e-06,
     3.660404672700679e-06,
     9.345512808067724e-06,
     1.3316865761225927e-06,
     2.7856904125656e-06,
     1.1878885743499268e-05,
     1.2834397011829424e-06,
     2.8480574201239506e-06,
     1.1490847100503743e-05,
     1.1321013744236552e-06,
     1.3817469834975782e-06,
     5.026918756811938e-07,
     2.3160316686698934e-07,
     1.0515557278267806e-06,
     5.254527195575065e-07,
     1.3514893453248078e-06,
     1.0991680028382689e-06,
     2.4212079097196693e-06,
     6.852813072555364e-08,
     4.1799285099841654e-07,
     1.1489362350403098e-06,
     1.911887466121698e-06,
     3.4247241842422227e-07,
     4.2735103988889023e-07,
     3.3010708193614846e-06,
     2.2823887775302865e-06,
     1.326952133240411e-05,
     5.452054097077053e-07,
     5.826502729178173e-07,
     1.6251367185304844e-07,
     1.2646340792343835e-06,
     1.0991680028382689e-06,
     2.51190300559756e-07,
     1.601529788786138e-06,
     1.7115230548370164e-06,
     8.615670594736002e-07,
     8.181858106581785e-07,
     8.363963388546836e-08,
     3.7143598774491693e-07,
     1.5895477645244682e-07,
     5.41196413905709e-07,
     6.460738859459525e-07,
     2.9990651455591433e-06,
     2.684765149751911e-06,
     8.24352537165396e-06,
     3.252701617384446e-06,
     2.5311601348221302e-05,
     1.6490901089127874e-08,
     4.600256886533316e-08,
     1.4654111701872807e-08,
     5.741129029956937e-07,
     5.657006454384828e-07,
     7.004433144608413e-10,
     2.492792416930456e-09,
     4.029007413919317e-06,
     1.6733997521001243e-09,
     6.319260705822671e-07,
     4.7026458283028205e-09,
     1.4336893627842073e-06,
     2.0359322661533952e-05,
     2.3160316686698934e-07,
     3.476097617749474e-06,
     1.3611515470302038e-08,
     1.4657872497991775e-06,
     2.5115806323583456e-08,
     6.956502602406545e-06,
     2.2157088608310005e-07,
     5.294130914990092e-06,
     7.324405828512681e-07,
     2.587828566902317e-05,
     1.3714105762119289e-07,
     5.372857685870258e-06,
     1.566459218338423e-06,
     5.870417226105928e-06,
     1.6740439150453312e-06,
     2.7246890113019617e-06,
     2.166911095002888e-08,
     3.25228427300317e-07,
     8.180808208635426e-08,
     1.3415514104053727e-06,
     1.4336893627842073e-06,
     2.0431959910638398e-06,
     3.7143598774491693e-07,
     1.998454081331147e-06,
     2.475732435414102e-05,
     2.645087135988433e-07,
     1.7369741271977546e-06,
     5.138139691496235e-09,
     5.493146545632044e-06,
     4.8454157877131365e-06,
     7.433322366523498e-07,
     5.027563929616008e-06,
     6.271183572437167e-09,
     5.913145173508383e-07,
     1.6371754441024677e-07,
     1.8022701624431647e-06,
     2.6643395045766738e-08,
     1.0324641152692493e-05,
     2.703959545158341e-08,
     2.420586575624384e-08,
     8.00064547945567e-09,
     1.1574471727726632e-06,
     3.6205707147018984e-05,
     1.371586563436722e-06,
     1.1070273586710755e-08,
     1.0133279459978439e-07,
     1.3415514104053727e-06,
     8.744910701352637e-06,
     2.6069987143273465e-05,
     1.0324641152692493e-05,
     2.765206090771244e-06,
     2.0061006580363028e-05,
     1.4546357007816368e-08,
     9.137354872734704e-09,
     2.215993163190433e-06,
     4.567601536109578e-06,
     3.5005001119969847e-09,
     2.1994159737914742e-07,
     3.157674086651241e-07,
     4.498942551833807e-09,
     5.2920885451612776e-09,
     8.42808458401123e-06,
     1.0827845287053606e-08,
     8.24246797037631e-07,
     1.183360723189253e-06,
     4.368066353066524e-09,
     1.2832751394853403e-07,
     1.8978286107085296e-06,
     3.349729240653687e-07,
     1.5097067489477922e-06,
     1.6253452486125752e-06,
     3.325524176034378e-06,
     1.0361476370235323e-06,
     4.368066353066524e-09,
     2.420586575624384e-08,
     4.3691875362128485e-07,
     1.0671927839212003e-06,
     3.2763765034360404e-07,
     8.303525760311459e-07,
     7.217084316835098e-07,
     1.312173935730243e-06,
     6.957395089557394e-05,
     8.2445825682953e-05,
     0.00019053347932640463,
     0.000124183323350735,
     1.686661380517762e-05,
     3.940779606637079e-06,
     3.6205707147018984e-05,
     4.5012562623014674e-05,
     5.084181611891836e-05,
     5.084181611891836e-05,
     2.457212076478754e-06,
     0.0006380286649800837,
     2.4848864995874465e-05,
     2.167744969483465e-05,
     6.853692866570782e-07,
     8.55341295391554e-06,
     1.538017750135623e-05,
     1.5899571735644713e-05,
     0.00028593215392902493,
     3.158079152854043e-06,
     1.8091644960804842e-05,
     1.625553886697162e-05,
     6.366072398122924e-07,
     2.088939936584211e-06,
     4.4015533262609097e-07,
     1.4126820815363317e-06,
     6.413230835278227e-07,
     1.0360147939536546e-07,
     1.4443097597904853e-06,
     7.656039997527841e-07,
     3.882538806010416e-07,
     1.9260503449913813e-06,
     1.6373854805351584e-06,
     1.4126820815363317e-06,
     5.783658707514405e-07,
     3.2763765034360404e-07,
     4.305167351503769e-07,
     1.8156210899178404e-06,
     8.002690492503461e-07,
     2.549583086874918e-06,
     4.434159279753658e-07,
     1.0991680028382689e-06,
     4.2735103988889023e-07,
     2.911820729423198e-06,
     1.8978286107085296e-06,
     1.3514893453248078e-06,
     2.2324086330627324e-06,
     9.344313411929761e-07,
     7.600717708555749e-06,
     5.698913128071581e-07,
     2.013257926591905e-06,
     4.953897132509155e-06,
     1.3025251064391341e-06,
     4.774418357555987e-06,
     0.009822138585150242,
     7.769888838993211e-07,
     4.500674549490213e-06,
     1.8700208102018223e-06,
     2.7856904125656e-06,
     3.6875201203656616e-06,
     2.8691551960946526e-06,
     3.770077910303371e-06,
     1.1921267741854535e-06,
     6.273598046391271e-06,
     3.9993806240090635e-06,
     7.490308053093031e-05,
     2.4666143872309476e-05,
     0.018862541764974594,
     0.003608161583542824,
     0.0070496490225195885,
     0.08074595779180527,
     0.008825290948152542,
     0.022112583741545677,
     0.012658679857850075,
     0.009349115192890167,
     0.024744026362895966,
     0.04498836770653725,
     0.07970976829528809,
     0.0031130046118050814,
     0.017039315775036812,
     0.059278689324855804,
     0.03922849893569946,
     0.027545301243662834,
     0.01124873198568821,
     0.04414552077651024,
     0.02785193920135498,
     0.018754635006189346,
     0.005899829789996147,
     0.005434760823845863,
     0.05824866518378258,
     0.015111787244677544,
     0.09974077343940735,
     0.006386995781213045,
     0.022665081545710564,
     0.005242695566266775,
     0.01581607013940811,
     0.0049239033833146095,
     0.05686810240149498,
     0.015947936102747917,
     0.10817603766918182,
     0.006020799744874239
    ],
    "p_oracle": [
     0.0001469101116526872,
     0.0001461399078834802,
     1.8727199858403765e-05,
     3.445339825702831e-05,
     1.491959483246319e-05,
     8.525082193955313e-06,
     2.203234726039227e-05,
     6.327398750727298e-06,
     2.4345908968825825e-06,
     4.6558663598261774e-05,
     3.151977125526173e-06,
     2.988307187479222e-06,
     1.7149484847323038e-05,
     2.3426230200129794e-06,
     3.3076789804908913e-06,
     8.558708032069262e-06,
     1.175259626506886e-06,
     2.548649490563548e-06,
     1.0875735824811272e-05,
     1.174447334051365e-06,
     2.7018331820727326e-06,
     1.073516978067346e-05,
     1.0747947953859693e-06,
     1.2419112636052887e-06,
     4.439075667050929e-07,
     2.042345670361101e-07,
     9.404049592376396e-07,
     4.902366299575078e-07,
     1.2479507631724118e-06,
     9.923292054736521e-07,
     2.2280037228483707e-06,
     5.986074569364064e-08,
     3.658147988971905e-07,
     1.0361229669797467e-06,
     1.7353581824863795e-06,
     3.016077982920251e-07,
     3.907217376308836e-07,
     2.925652097474085e-06,
     1.966859144886257e-06,
     1.2392310054565314e-05,
     5.118952799421095e-07,
     5.359339070309943e-07,
     1.4427850203446724e-07,
     1.123934225688572e-06,
     1.0238458116873517e-06,
     2.1720221354826208e-07,
     1.4275401554186828e-06,
     1.5700845779065276e-06,
     8.077286679508688e-07,
     7.595568831675337e-07,
     7.496300469256312e-08,
     3.4219951317027153e-07,
     1.4025690120433865e-07,
     5.061549472884508e-07,
     5.872788619853964e-07,
     2.820014060489484e-06,
     2.5274971449107397e-06,
     7.469083357136697e-06,
     2.887191612899187e-06,
     2.2984444512985647e-05,
     1.4499901723752373e-08,
     4.224759564408487e-08,
     1.3127833931037003e-08,
     5.267225446914381e-07,
     5.032041485719674e-07,
     5.929003399351984e-10,
     2.150327471639457e-09,
     3.6878504943160806e-06,
     1.5137613385007853e-09,
     5.771068458670925e-07,
     4.05874622799729e-09,
     1.283262236029259e-06,
     1.8895223547588103e-05,
     2.1562215124504291e-07,
     3.2647783427819377e-06,
     1.216329259534632e-08,
     1.3845196917827707e-06,
     2.249413455501781e-08,
     6.356727226375369e-06,
     2.0436063152828865e-07,
     4.80034123029327e-06,
     6.884560548314766e-07,
     2.4657274479977787e-05,
     1.261227424720346e-07,
     5.058232545707142e-06,
     1.4855725112283835e-06,
     5.549032721319236e-06,
     1.5243693951561e-06,
     2.5501326490484644e-06,
     1.9503026749134733e-08,
     2.894741442105442e-07,
     7.430752901882443e-08,
     1.2425402928784024e-06,
     1.359587031402043e-06,
     1.943861207109876e-06,
     3.3811633670666197e-07,
     1.8900060467785806e-06,
     2.3787959435139783e-05,
     2.389311362094304e-07,
     1.6118586927404976e-06,
     4.5472798859691466e-09,
     5.195611720409943e-06,
     4.406738753459649e-06,
     6.553700018230302e-07,
     4.563604306895286e-06,
     5.59060486793328e-09,
     5.400765417107323e-07,
     1.514987673090218e-07,
     1.6380289480366628e-06,
     2.4197854386898143e-08,
     9.32275997911347e-06,
     2.393656828303392e-08,
     2.1264302318968475e-08,
     6.9980532479974045e-09,
     1.081376012734836e-06,
     3.306658618384972e-05,
     1.222553464685916e-06,
     9.822396229708374e-09,
     9.009193746578603e-08,
     1.2590995766004198e-06,
     7.929240382509306e-06,
     2.503086216165684e-05,
     9.561872502672486e-06,
     2.482203854015097e-06,
     1.9406297724344768e-05,
     1.3104779483796847e-08,
     8.068131940319745e-09,
     2.05597734748153e-06,
     4.0540408008382656e-06,
     3.169926543478141e-09,
     1.9746653379115742e-07,
     2.882631235934241e-07,
     3.9420209319018795e-09,
     4.785592810918615e-09,
     8.075558071141131e-06,
     9.4671372963262e-09,
     7.656602178940375e-07,
     1.0820309626069502e-06,
     3.729021536003074e-09,
     1.2206049859742052e-07,
     1.735182763695775e-06,
     3.0206345513761335e-07,
     1.4078510730541893e-06,
     1.526253186057147e-06,
     3.32214199261216e-06,
     9.722509730636375e-07,
     3.903593004395134e-09,
     2.2041644953674222e-08,
     4.0664875200491224e-07,
     9.705481716082431e-07,
     2.8843609811701754e-07,
     7.440521017088031e-07,
     6.660162625848898e-07,
     1.2110650686736335e-06,
     6.550806574523449e-05,
     7.979811925906688e-05,
     0.0001818382297642529,
     0.00011510254262248054,
     1.574801717652008e-05,
     3.7079730645928066e-06,
     3.386830576346256e-05,
     4.226343662594445e-05,
     4.857313251704909e-05,
     4.7070567234186456e-05,
     2.2569599877897417e-06,
     0.0006120369071140885,
     2.3661097657168284e-05,
     2.0005791157018393e-05,
     6.244418386813777e-07,
     7.96031963545829e-06,
     1.4662005014542956e-05,
     1.4956691302359104e-05,
     0.0002772198058664799,
     2.891156555051566e-06,
     1.6640678950352594e-05,
     1.5442494259332307e-05,
     5.737019250773301e-07,
     1.9168603557773167e-06,
     3.9845309629527037e-07,
     1.2928065871165018e-06,
     5.787465511275514e-07,
     9.430754488448656e-08,
     1.3109068959238357e-06,
     6.849967917332833e-07,
     3.59871819455293e-07,
     1.7630660522627295e-06,
     1.5343573522841325e-06,
     1.2713866226476966e-06,
     5.288820261739602e-07,
     2.9492520070562023e-07,
     3.9059952428033284e-07,
     1.6423742863480584e-06,
     7.348767780968046e-07,
     2.3556704036309384e-06,
     4.0035379811342864e-07,
     9.642487839300884e-07,
     3.915128559128789e-07,
     2.6290808818885125e-06,
     1.7516496200187248e-06,
     1.2177035841887118e-06,
     2.065924718408496e-06,
     8.670426154822053e-07,
     6.833991392340977e-06,
     4.97766279750067e-07,
     1.818960640775913e-06,
     4.63209062218084e-06,
     1.1824243983937777e-06,
     4.360892944532679e-06,
     0.009755837731063366,
     7.05798640865396e-07,
     4.1220273487851955e-06,
     1.7139896044682246e-06,
     2.5792653559619794e-06,
     3.3125340905826306e-06,
     2.6990644528268604e-06,
     3.3724088552844478e-06,
     1.0858207133424003e-06,
     5.779492767032934e-06,
     3.6232527236279566e-06,
     6.950746319489554e-05,
     2.359280188102275e-05,
     0.018962303176522255,
     0.0035667235497385263,
     0.006895596627146006,
     0.08383864909410477,
     0.008625222370028496,
     0.022189911454916,
     0.012730501592159271,
     0.009463458321988583,
     0.024959232658147812,
     0.0444105863571167,
     0.08021362125873566,
     0.0028592399321496487,
     0.017242614179849625,
     0.060589153319597244,
     0.039358872920274734,
     0.02689761482179165,
     0.011007904075086117,
     0.044373709708452225,
     0.02796688862144947,
     0.0190079715102911,
     0.005807948764413595,
     0.005553585942834616,
     0.05795270577073097,
     0.014917745254933834,
     0.0977199524641037,
     0.006410952657461166,
     0.022017937153577805,
     0.005312574561685324,
     0.015614868141710758,
     0.004747352562844753,
     0.057182542979717255,
     0.01578008383512497,
     0.107459157705307,
     0.005929228384047747
    ],
    "abstain": 0.00023595143284182996,
    "abstain_oracle": 0.00022339480347000062,
    "abs_delta_abstain": 1.2556629371829331e-05,
    "oracle_margin": 0.009739205241203308,
    "max_abs_delta_p": 0.0030926913022994995,
    "mean_abs_delta_p": 5.445285578352905e-05,
    "max_abs_delta_raw_logits": 0.17230606079101562,
    "mean_abs_delta_raw_logits": 0.07908498687902465,
    "logits_min": -18.140625,
    "logits_max": 1.818359375,
    "raw_logits_min": -18.140625,
    "raw_logits_max": 1.818359375,
    "raw_logits_absmax": 18.140625,
    "oracle_logits_min": -18.312931060791016,
    "oracle_logits_max": 1.8154594898223877,
    "oracle_raw_logits_absmax": 18.312931060791016,
    "finite": true,
    "nonconstant": true,
    "wall_seconds_contended": 8.328881041961722,
    "seconds": 8.328881041961722,
    "full_vocab_argmax_id": 253,
    "full_vocab_argmax_is_label": true
   },
   {
    "id": "r16-level",
    "request_id": "r16",
    "question_id": "level",
    "type": "score",
    "qtype_index": 1,
    "kind": "slot",
    "tokens": 160,
    "nopts": 10,
    "temperature": 1.0431410074234009,
    "raw_logits": [
     -1.1787109375,
     -2.443359375,
     -2.10546875,
     3.326171875,
     -1.439453125,
     -2.927734375,
     -3.333984375,
     -3.87109375,
     -4.4609375,
     -1.46484375,
     -1.50390625,
     -2.982421875,
     -3.341796875,
     -1.501953125,
     -3.517578125,
     -4.515625,
     -4.3828125,
     -5.046875,
     -2.677734375,
     -2.7578125,
     -3.75390625,
     -1.6845703125,
     -5.70703125,
     -5.375,
     -3.931640625,
     -4.3046875,
     -2.759765625,
     -4.8125,
     -3.49609375,
     -4.765625,
     -4.21875,
     -4.23046875,
     -6.234375,
     -2.94140625,
     -1.9638671875,
     -4.76953125,
     -4.921875,
     -5.6484375,
     -3.865234375,
     -3.140625,
     -6.6875,
     -4.1015625,
     -5.03515625,
     -5.86328125,
     -3.470703125,
     -4.2265625,
     -4.43359375,
     -4.859375,
     -5.4140625,
     -3.939453125,
     -4.46484375,
     -4.046875,
     -2.85546875,
     -5.63671875,
     -2.328125,
     -2.525390625,
     -4.3203125,
     -4.7265625,
     -4.36328125,
     -5.82421875,
     -7.08203125,
     -6.66015625,
     -7.7265625,
     -5.64453125,
     -3.474609375,
     -6.80859375,
     -7.3671875,
     -1.7509765625,
     -7.3984375,
     -4.46875,
     -7.66015625,
     -5.15625,
     -2.416015625,
     -5.09765625,
     -3.6953125,
     -5.625,
     -4.43359375,
     -6.54296875,
     -3.5,
     -5.2265625,
     -4.19921875,
     -3.599609375,
     -2.345703125,
     -3.94921875,
     -3.087890625,
     -4.1171875,
     -2.556640625,
     -2.986328125,
     -3.267578125,
     -4.30859375,
     -3.646484375,
     -4.09765625,
     -3.8359375,
     -3.4921875,
     -4.54296875,
     -4.61328125,
     -4.78515625,
     -3.337890625,
     -3.384765625,
     -3.732421875,
     -5.06640625,
     -5.203125,
     -3.0390625,
     -3.6875,
     -4.45703125,
     -5.2890625,
     -3.474609375,
     -4.40625,
     -4.37890625,
     -5.5078125,
     -4.26171875,
     -4.75,
     -4.30078125,
     -5.54296875,
     -4.87109375,
     -3.927734375,
     -4.046875,
     -5.4765625,
     -3.712890625,
     -2.83984375,
     -2.69921875,
     -4.1953125,
     -3.146484375,
     -3.71875,
     -1.98828125,
     -5.15234375,
     -5.16796875,
     -2.65625,
     -3.4765625,
     -5.4453125,
     -5.30859375,
     -2.349609375,
     -5.8125,
     -5.9375,
     -4.3984375,
     -4.26171875,
     -3.7890625,
     -4.73046875,
     -5.7265625,
     -2.27734375,
     -3.044921875,
     -4.25,
     -3.111328125,
     -3.05859375,
     -4.0546875,
     -4.625,
     -5.75,
     -5.5234375,
     -5.60546875,
     -4.46875,
     -4.2109375,
     -2.615234375,
     -1.279296875,
     -3.55859375,
     -2.419921875,
     -1.4921875,
     -1.7822265625,
     -1.6796875,
     -2.66796875,
     -1.134765625,
     -2.279296875,
     -2.62890625,
     -1.556640625,
     -1.005859375,
     -3.40234375,
     -1.6904296875,
     -2.560546875,
     -2.263671875,
     -3.953125,
     -2.046875,
     -1.818359375,
     -2.171875,
     -2.177734375,
     -3.244140625,
     -1.0634765625,
     -1.8037109375,
     -3.359375,
     -3.048828125,
     -2.380859375,
     -2.80859375,
     -3.13671875,
     -2.318359375,
     -2.66796875,
     -1.71875,
     -2.818359375,
     -1.9580078125,
     -1.5537109375,
     -2.044921875,
     -2.005859375,
     -1.697265625,
     -0.982421875,
     -1.392578125,
     -2.421875,
     -1.48828125,
     -1.4248046875,
     -2.1484375,
     -1.501953125,
     -1.28515625,
     -1.94921875,
     -0.99560546875,
     -1.611328125,
     -1.5859375,
     -1.2314453125,
     -2.00390625,
     -2.04296875,
     -1.19921875,
     -1.9072265625,
     -2.740234375,
     0.25390625,
     -2.099609375,
     -0.72607421875,
     -1.310546875,
     -0.54541015625,
     -1.08203125,
     -2.330078125,
     -0.97021484375,
     -1.677734375,
     -2.0546875,
     -1.30078125,
     -1.68359375,
     -1.404296875,
     -1.6123046875,
     -1.0830078125,
     -1.4873046875,
     -0.08636474609375,
     0.1998291015625,
     -0.830078125,
     0.80859375,
     1.056640625,
     -0.478271484375,
     -0.927734375,
     -0.5146484375,
     -0.45947265625,
     -1.099609375,
     0.5634765625,
     0.6708984375,
     1.26171875,
     -0.53564453125,
     -0.473388671875,
     -0.487060546875,
     0.69921875,
     0.4296875,
     -0.7705078125,
     0.286865234375,
     -0.0771484375,
     0.09771728515625,
     0.2841796875,
     -0.466796875,
     -0.18408203125,
     1.791015625,
     -0.11688232421875,
     -0.62646484375,
     -0.5634765625,
     1.1103515625,
     -0.1773681640625,
     -5.34375
    ],
    "raw256_argmax": 3,
    "option_argmax": 3,
    "oracle_argmax": 3,
    "argmax_agrees": true,
    "p_full": [
     0.012693560682237148,
     0.003776311641559005,
     0.005220871418714523,
     0.953052282333374,
     0.009886160492897034,
     0.002373594092205167,
     0.001607944373972714,
     0.0009608473628759384,
     0.0005458617233671248,
     0.009648430161178112,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.00023417382908519357
    ],
    "p": [
     0.012696533463895321,
     0.003777195932343602,
     0.0052220942452549934,
     0.9532754421234131,
     0.009888475760817528,
     0.0023741498589515686,
     0.001608320977538824,
     0.0009610723936930299,
     0.0005459895473904908,
     0.00965068954974413
    ],
    "p_oracle": [
     0.011903850361704826,
     0.0036331256851553917,
     0.0049773394130170345,
     0.9555612802505493,
     0.009435553103685379,
     0.0023825846146792173,
     0.00155331299174577,
     0.0009360660333186388,
     0.0005382035742513835,
     0.009078679606318474
    ],
    "abstain": 0.00023417382908519357,
    "abstain_oracle": 0.00022859007003717124,
    "abs_delta_abstain": 5.58375904802233e-06,
    "oracle_margin": 0.9436574578285217,
    "max_abs_delta_p": 0.0022858381271362305,
    "mean_abs_delta_p": 0.00045885139843448997,
    "max_abs_delta_raw_logits": 0.04090452194213867,
    "mean_abs_delta_raw_logits": 0.0067774298950098455,
    "logits_min": -7.7265625,
    "logits_max": 3.326171875,
    "raw_logits_min": -7.7265625,
    "raw_logits_max": 3.326171875,
    "raw_logits_absmax": 7.7265625,
    "oracle_logits_min": -7.726421356201172,
    "oracle_logits_max": 3.365875244140625,
    "oracle_raw_logits_absmax": 7.726421356201172,
    "finite": true,
    "nonconstant": true,
    "wall_seconds_contended": 0.8945889160968363,
    "seconds": 0.8945889160968363,
    "full_vocab_argmax_id": 3,
    "full_vocab_argmax_is_label": true
   }
  ]
 },
 "reset_check": {
  "row": "r01-route",
  "identical": true,
  "max_abs_diff": 0.0,
  "final_session": "int8lin-04-global_reset-a1",
  "final_raw_logits": [
   0.923828125,
   -4.08984375,
   -1.787109375,
   -3.15234375,
   -3.03515625,
   -3.1484375,
   -3.1015625,
   -3.046875,
   -4.265625,
   -2.974609375,
   -3.037109375,
   -2.724609375,
   -4.38671875,
   -3.40234375,
   -3.67578125,
   -3.583984375,
   -3.9765625,
   -4.8984375,
   -3.580078125,
   -3.421875,
   -3.69140625,
   -2.41015625,
   -5.51171875,
   -5.21875,
   -3.9375,
   -4.7890625,
   -3.427734375,
   -4.73828125,
   -4.14453125,
   -5.67578125,
   -4.3125,
   -4.5390625,
   -5.82421875,
   -3.943359375,
   -3.287109375,
   -4.94140625,
   -5.75,
   -6.21484375,
   -4.6875,
   -3.01953125,
   -5.88671875,
   -4.8046875,
   -5.15234375,
   -5.96484375,
   -3.826171875,
   -4.96484375,
   -4.35546875,
   -4.94921875,
   -5.39453125,
   -4.703125,
   -5.26953125,
   -4.6328125,
   -3.884765625,
   -5.47265625,
   -3.40625,
   -3.783203125,
   -3.955078125,
   -4.4453125,
   -4.58203125,
   -5.77734375,
   -7.453125,
   -7.21484375,
   -8.140625,
   -5.8203125,
   -3.728515625,
   -7.51171875,
   -8.03125,
   -2.712890625,
   -7.87109375,
   -4.35546875,
   -8.0078125,
   -5.390625,
   -2.765625,
   -5.41796875,
   -3.7734375,
   -6.3828125,
   -4.125,
   -7.0390625,
   -4.0390625,
   -6.0703125,
   -4.66015625,
   -4.265625,
   -3.19140625,
   -4.75,
   -3.564453125,
   -4.89453125,
   -2.65625,
   -4.12109375,
   -3.328125,
   -4.48828125,
   -4.02734375,
   -4.14453125,
   -4.31640625,
   -3.615234375,
   -4.78515625,
   -4.1171875,
   -5.1015625,
   -3.16015625,
   -2.66796875,
   -4.76171875,
   -6.18359375,
   -4.96875,
   -3.4140625,
   -4.359375,
   -4.7890625,
   -5.94921875,
   -3.48046875,
   -5.23828125,
   -4.140625,
   -6.0234375,
   -4.1796875,
   -4.98046875,
   -5.04296875,
   -5.79296875,
   -4.703125,
   -3.51171875,
   -4.7890625,
   -6.01953125,
   -4.265625,
   -3.701171875,
   -3.140625,
   -4.2890625,
   -2.89453125,
   -3.689453125,
   -2.212890625,
   -5.1171875,
   -5.84375,
   -2.671875,
   -2.82421875,
   -5.80078125,
   -5.6875,
   -3.255859375,
   -5.8203125,
   -6.65625,
   -5.3671875,
   -4.9609375,
   -3.77734375,
   -4.33984375,
   -6.3671875,
   -2.70703125,
   -3.63671875,
   -4.328125,
   -3.044921875,
   -2.8046875,
   -4.25390625,
   -4.93359375,
   -5.8984375,
   -5.80078125,
   -5.22265625,
   -4.51953125,
   -4.5546875,
   -2.87109375,
   -1.0556640625,
   -3.166015625,
   -1.939453125,
   -1.4638671875,
   -1.576171875,
   -1.51953125,
   -2.634765625,
   -0.87158203125,
   -1.861328125,
   -1.751953125,
   -1.4228515625,
   -1.3427734375,
   -2.6953125,
   -1.84765625,
   -2.994140625,
   -2.28515625,
   -3.892578125,
   -2.1171875,
   -2.21875,
   -2.548828125,
   -1.87890625,
   -2.92578125,
   -0.955078125,
   -1.955078125,
   -3.685546875,
   -3.427734375,
   -3.251953125,
   -2.583984375,
   -3.9296875,
   -2.513671875,
   -3.091796875,
   -2.76953125,
   -3.40625,
   -2.51953125,
   -2.005859375,
   -2.0625,
   -2.54296875,
   -2.66796875,
   -1.275390625,
   -2.015625,
   -2.771484375,
   -1.85546875,
   -2.064453125,
   -2.109375,
   -2.08203125,
   -2.03515625,
   -2.125,
   -1.1337890625,
   -1.404296875,
   -2.62890625,
   -1.26953125,
   -2.28125,
   -2.572265625,
   -1.7861328125,
   -2.2109375,
   -2.498046875,
   0.06658935546875,
   -2.31640625,
   -1.7529296875,
   -1.2412109375,
   -0.11883544921875,
   -1.1279296875,
   -3.3125,
   -1.12890625,
   -2.02734375,
   -2.14453125,
   -1.5546875,
   -2.111328125,
   -1.2255859375,
   -1.3544921875,
   -1.0224609375,
   -0.8388671875,
   -0.1019287109375,
   -0.44970703125,
   -0.771484375,
   0.9013671875,
   0.74267578125,
   -0.39111328125,
   -0.036651611328125,
   -0.456298828125,
   0.11053466796875,
   -0.9267578125,
   0.09326171875,
   1.0791015625,
   0.89404296875,
   -0.0767822265625,
   -0.09173583984375,
   -0.7197265625,
   0.303466796875,
   0.54296875,
   -0.71826171875,
   0.053619384765625,
   -0.140380859375,
   0.416748046875,
   0.7802734375,
   -0.76318359375,
   -0.1309814453125,
   2.05859375,
   -0.72607421875,
   -0.27685546875,
   -0.50244140625,
   0.0005345344543457031,
   -0.30810546875,
   -2.701171875
  ],
  "cross_process": true,
  "local_resets_in_every_session": true
 },
 "wall_seconds_contended": 65.2358645000495,
 "completed_at": "2026-09-22T19:28:41+00:00",
 "generated_at": "2026-09-22T19:28:41+00:00"
}