{
 "schema": "coreai-decider-readout-gate/1",
 "head": "slot",
 "mode": "fp16",
 "bundle": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/openthai_systemone_decode_fp16",
 "fixtures": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/models/openthai-systemone/fixtures-openthai-systemone.json",
 "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
 "max_ctx": 4096,
 "runtime": "coreai Python runtime, AOT h16c GPU asset, SpecializationOptions.default()",
 "source": {
  "hf_id": "iapp/OpenThai-SystemOne",
  "revision": "f3709948b5e3cc9606a57e74ba62b7a639d17dd3",
  "oracle": "author's modeling.py slot_logits, fp32, CPU",
  "transformers": "5.17.0",
  "torch": "2.9.0",
  "client_revision": "5d04bcc",
  "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/snapshots/f3709948b5e3cc9606a57e74ba62b7a639d17dd3"
 },
 "temperature_by_type": {
  "choice": 1.058534,
  "score": 1.043141,
  "noul": 1.006767
 },
 "n_slots": 256,
 "abstain_slot": 255,
 "answer_token_id": 248082,
 "environment": {
  "python": "3.11.13",
  "platform": "darwin",
  "DEVELOPER_DIR": "/Applications/Xcode-27.0.0-RC.app/Contents/Developer"
 },
 "started_at": "2026-09-22T19:26:27+00:00",
 "timing": "contended; correctness only, no performance claim",
 "rows": [
  {
   "id": "r01-route",
   "request_id": "r01",
   "question_id": "route",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 113,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    0.93798828125,
    -4.04296875,
    -1.77734375,
    -3.1484375,
    -3.025390625,
    -3.1171875,
    -3.091796875,
    -3.0546875,
    -4.25,
    -2.951171875,
    -3.03125,
    -2.72265625,
    -4.34375,
    -3.388671875,
    -3.658203125,
    -3.583984375,
    -3.96484375,
    -4.8984375,
    -3.572265625,
    -3.408203125,
    -3.673828125,
    -2.38671875,
    -5.4921875,
    -5.2109375,
    -3.927734375,
    -4.7734375,
    -3.419921875,
    -4.7265625,
    -4.140625,
    -5.67578125,
    -4.296875,
    -4.5234375,
    -5.80859375,
    -3.939453125,
    -3.271484375,
    -4.9296875,
    -5.73046875,
    -6.21484375,
    -4.6953125,
    -3.015625,
    -5.89453125,
    -4.79296875,
    -5.14453125,
    -5.94921875,
    -3.826171875,
    -4.9609375,
    -4.34375,
    -4.94921875,
    -5.390625,
    -4.69140625,
    -5.26953125,
    -4.62109375,
    -3.86328125,
    -5.46875,
    -3.400390625,
    -3.7734375,
    -3.9453125,
    -4.4375,
    -4.578125,
    -5.76953125,
    -7.4453125,
    -7.1953125,
    -8.125,
    -5.8125,
    -3.712890625,
    -7.5,
    -8.015625,
    -2.70703125,
    -7.85546875,
    -4.3515625,
    -7.98828125,
    -5.375,
    -2.759765625,
    -5.4140625,
    -3.76171875,
    -6.3671875,
    -4.109375,
    -7.03125,
    -4.02734375,
    -6.0546875,
    -4.65625,
    -4.25390625,
    -3.181640625,
    -4.75,
    -3.55859375,
    -4.8828125,
    -2.650390625,
    -4.10546875,
    -3.326171875,
    -4.47265625,
    -4.01171875,
    -4.15234375,
    -4.30859375,
    -3.6171875,
    -4.76953125,
    -4.1015625,
    -5.0859375,
    -3.162109375,
    -2.6640625,
    -4.7421875,
    -6.1640625,
    -4.94921875,
    -3.408203125,
    -4.33984375,
    -4.77734375,
    -5.9453125,
    -3.4765625,
    -5.23046875,
    -4.12890625,
    -6.015625,
    -4.171875,
    -4.95703125,
    -5.01953125,
    -5.78125,
    -4.6953125,
    -3.5,
    -4.77734375,
    -5.984375,
    -4.2578125,
    -3.68359375,
    -3.125,
    -4.28515625,
    -2.890625,
    -3.681640625,
    -2.197265625,
    -5.09765625,
    -5.828125,
    -2.65625,
    -2.80078125,
    -5.77734375,
    -5.67578125,
    -3.236328125,
    -5.8125,
    -6.62890625,
    -5.34765625,
    -4.94140625,
    -3.755859375,
    -4.33203125,
    -6.3515625,
    -2.697265625,
    -3.625,
    -4.32421875,
    -3.037109375,
    -2.794921875,
    -4.24609375,
    -4.91796875,
    -5.87890625,
    -5.78515625,
    -5.21484375,
    -4.51171875,
    -4.5546875,
    -2.853515625,
    -1.0400390625,
    -3.15234375,
    -1.9384765625,
    -1.4453125,
    -1.5673828125,
    -1.51953125,
    -2.634765625,
    -0.86865234375,
    -1.8623046875,
    -1.7509765625,
    -1.4150390625,
    -1.34765625,
    -2.693359375,
    -1.841796875,
    -2.9765625,
    -2.271484375,
    -3.876953125,
    -2.115234375,
    -2.2109375,
    -2.541015625,
    -1.865234375,
    -2.91796875,
    -0.94873046875,
    -1.951171875,
    -3.669921875,
    -3.4140625,
    -3.232421875,
    -2.56640625,
    -3.90625,
    -2.49609375,
    -3.078125,
    -2.763671875,
    -3.390625,
    -2.501953125,
    -2.0,
    -2.05078125,
    -2.541015625,
    -2.658203125,
    -1.2587890625,
    -2.01171875,
    -2.76953125,
    -1.8408203125,
    -2.048828125,
    -2.10546875,
    -2.076171875,
    -2.02734375,
    -2.1171875,
    -1.134765625,
    -1.3955078125,
    -2.607421875,
    -1.2646484375,
    -2.27734375,
    -2.55859375,
    -1.7783203125,
    -2.203125,
    -2.5,
    0.07110595703125,
    -2.3203125,
    -1.7548828125,
    -1.232421875,
    -0.10736083984375,
    -1.1083984375,
    -3.306640625,
    -1.12890625,
    -2.017578125,
    -2.140625,
    -1.548828125,
    -2.09765625,
    -1.2158203125,
    -1.3671875,
    -1.0185546875,
    -0.8388671875,
    -0.0960693359375,
    -0.44384765625,
    -0.76806640625,
    0.890625,
    0.74658203125,
    -0.3798828125,
    -0.0428466796875,
    -0.452392578125,
    0.1131591796875,
    -0.91845703125,
    0.08892822265625,
    1.0849609375,
    0.904296875,
    -0.081298828125,
    -0.1031494140625,
    -0.72021484375,
    0.312255859375,
    0.53759765625,
    -0.7060546875,
    0.048126220703125,
    -0.137939453125,
    0.408935546875,
    0.78076171875,
    -0.75927734375,
    -0.13330078125,
    2.06640625,
    -0.72412109375,
    -0.288818359375,
    -0.50244140625,
    -0.0106964111328125,
    -0.3037109375,
    -2.74609375
   ],
   "raw256_argmax": 249,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.8954583406448364,
    0.008099568076431751,
    0.06886537373065948,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.027576737105846405
   ],
   "p": [
    0.9208524823188782,
    0.008329262025654316,
    0.07081831246614456
   ],
   "p_oracle": [
    0.9192458987236023,
    0.008343691937625408,
    0.072410449385643
   ],
   "abstain": 0.027576737105846405,
   "abstain_oracle": 0.02907981537282467,
   "abs_delta_abstain": 0.0015030782669782639,
   "oracle_margin": 0.8468354344367981,
   "max_abs_delta_p": 0.001606583595275879,
   "mean_abs_delta_p": 0.0010710501422484715,
   "max_abs_delta_raw_logits": 0.03141129016876221,
   "mean_abs_delta_raw_logits": 0.00686514304834418,
   "logits_min": -8.125,
   "logits_max": 2.06640625,
   "raw_logits_min": -8.125,
   "raw_logits_max": 2.06640625,
   "raw_logits_absmax": 8.125,
   "oracle_logits_min": -8.13015365600586,
   "oracle_logits_max": 2.0633702278137207,
   "oracle_raw_logits_absmax": 8.13015365600586,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 1.8779243340250105,
   "seconds": 1.8779243340250105,
   "full_vocab_argmax_id": 249,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r01-condition",
   "request_id": "r01",
   "question_id": "condition",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 87,
   "nopts": 4,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -3.24609375,
    3.130859375,
    -2.6171875,
    -0.74267578125,
    -2.30078125,
    -3.001953125,
    -2.83984375,
    -3.623046875,
    -4.10546875,
    -3.08984375,
    -3.048828125,
    -3.9765625,
    -5.40234375,
    -3.068359375,
    -4.16796875,
    -4.73046875,
    -4.15234375,
    -4.44921875,
    -2.947265625,
    -2.98046875,
    -3.341796875,
    -1.6533203125,
    -5.2578125,
    -6.63671875,
    -4.109375,
    -5.39453125,
    -2.58984375,
    -5.8203125,
    -4.40234375,
    -6.65625,
    -4.67578125,
    -4.875,
    -6.25,
    -4.609375,
    -2.935546875,
    -5.47265625,
    -5.1640625,
    -5.6171875,
    -4.01171875,
    -2.943359375,
    -7.26953125,
    -4.25390625,
    -4.93359375,
    -5.90234375,
    -4.42578125,
    -5.07421875,
    -4.91796875,
    -4.80859375,
    -5.3046875,
    -4.60546875,
    -5.69140625,
    -4.8515625,
    -2.439453125,
    -5.05859375,
    -2.44140625,
    -3.08203125,
    -3.759765625,
    -4.49609375,
    -4.72265625,
    -5.73046875,
    -7.65234375,
    -6.4453125,
    -7.5859375,
    -6.55859375,
    -3.751953125,
    -7.55078125,
    -7.59765625,
    -2.166015625,
    -7.74609375,
    -4.26171875,
    -8.171875,
    -5.37109375,
    -2.5546875,
    -6.31640625,
    -3.861328125,
    -6.36328125,
    -4.38671875,
    -6.67578125,
    -4.13671875,
    -5.5703125,
    -4.1875,
    -4.046875,
    -2.46875,
    -3.966796875,
    -3.40625,
    -4.0234375,
    -2.720703125,
    -3.62890625,
    -3.056640625,
    -4.87109375,
    -3.578125,
    -4.1328125,
    -4.20703125,
    -4.34765625,
    -4.71875,
    -4.94140625,
    -5.203125,
    -4.109375,
    -4.14453125,
    -3.912109375,
    -6.00390625,
    -4.8828125,
    -3.48046875,
    -4.359375,
    -4.37890625,
    -6.21484375,
    -4.34375,
    -5.66015625,
    -4.29296875,
    -5.32421875,
    -4.45703125,
    -4.79296875,
    -5.1875,
    -6.59375,
    -4.93359375,
    -4.30078125,
    -4.6171875,
    -6.109375,
    -4.60546875,
    -3.333984375,
    -3.1640625,
    -4.12109375,
    -3.314453125,
    -3.841796875,
    -2.52734375,
    -5.953125,
    -5.78515625,
    -3.37109375,
    -3.267578125,
    -5.515625,
    -5.5234375,
    -3.7890625,
    -5.7578125,
    -6.08984375,
    -5.07421875,
    -4.74609375,
    -4.05078125,
    -5.03125,
    -5.8828125,
    -2.94921875,
    -3.73828125,
    -4.1953125,
    -3.78515625,
    -3.5234375,
    -4.796875,
    -5.27734375,
    -6.2890625,
    -5.26953125,
    -4.88671875,
    -4.40234375,
    -4.78125,
    -3.1484375,
    -1.3310546875,
    -3.67578125,
    -2.64453125,
    -1.798828125,
    -2.021484375,
    -2.154296875,
    -3.423828125,
    -0.791015625,
    -2.40234375,
    -2.259765625,
    -1.751953125,
    -1.35546875,
    -2.78125,
    -2.400390625,
    -2.865234375,
    -1.7333984375,
    -3.369140625,
    -2.197265625,
    -2.0859375,
    -2.59375,
    -2.51171875,
    -3.12109375,
    -1.162109375,
    -2.013671875,
    -3.25,
    -3.41796875,
    -3.0859375,
    -3.646484375,
    -3.546875,
    -3.271484375,
    -2.693359375,
    -2.021484375,
    -2.638671875,
    -1.5185546875,
    -1.40625,
    -1.787109375,
    -1.927734375,
    -1.896484375,
    -1.212890625,
    -1.689453125,
    -3.013671875,
    -1.5771484375,
    -1.3349609375,
    -2.138671875,
    -1.9267578125,
    -1.8916015625,
    -1.73046875,
    -1.3154296875,
    -1.3759765625,
    -1.951171875,
    -1.908203125,
    -1.7978515625,
    -2.353515625,
    -1.1259765625,
    -2.6640625,
    -2.734375,
    -0.384033203125,
    -2.302734375,
    -0.7998046875,
    -1.333984375,
    -0.82421875,
    -1.494140625,
    -2.447265625,
    -1.4921875,
    -1.8623046875,
    -2.443359375,
    -1.86328125,
    -2.0390625,
    -1.392578125,
    -1.736328125,
    -1.5703125,
    -1.416015625,
    -0.1927490234375,
    0.2607421875,
    -0.342529296875,
    0.73779296875,
    0.94677734375,
    -0.10662841796875,
    -1.0830078125,
    -0.492431640625,
    -0.1053466796875,
    -0.91259765625,
    0.85302734375,
    1.13671875,
    1.552734375,
    -0.6640625,
    0.057373046875,
    -0.76806640625,
    0.32763671875,
    0.6875,
    -0.775390625,
    -0.0853271484375,
    0.144775390625,
    0.54736328125,
    0.2430419921875,
    -0.324951171875,
    -0.65185546875,
    1.4921875,
    0.042083740234375,
    -0.744140625,
    -0.421630859375,
    0.5478515625,
    -0.5576171875,
    -4.60546875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0023413978051394224,
    0.967847466468811,
    0.004241333808749914,
    0.024921584874391556,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0006482636672444642
   ],
   "p": [
    0.0023429165594279766,
    0.9684752225875854,
    0.004244084935635328,
    0.024937748908996582
   ],
   "p_oracle": [
    0.0023138143587857485,
    0.9696792364120483,
    0.004179855342954397,
    0.023827005177736282
   ],
   "abstain": 0.0006482636672444642,
   "abstain_oracle": 0.000638672907371074,
   "abs_delta_abstain": 9.590759873390198e-06,
   "oracle_margin": 0.9458522200584412,
   "max_abs_delta_p": 0.0012040138244628906,
   "mean_abs_delta_p": 0.0006020223372615874,
   "max_abs_delta_raw_logits": 0.03208816051483154,
   "mean_abs_delta_raw_logits": 0.004462839162442833,
   "logits_min": -8.171875,
   "logits_max": 3.130859375,
   "raw_logits_min": -8.171875,
   "raw_logits_max": 3.130859375,
   "raw_logits_absmax": 8.171875,
   "oracle_logits_min": -8.181787490844727,
   "oracle_logits_max": 3.1483166217803955,
   "oracle_raw_logits_absmax": 8.181787490844727,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4949088330613449,
   "seconds": 0.4949088330613449,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r01-refund",
   "request_id": "r01",
   "question_id": "refund",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 87,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    3.20703125,
    -2.435546875,
    -2.130859375,
    -2.78125,
    -2.435546875,
    -2.66015625,
    -2.953125,
    -3.90234375,
    -3.216796875,
    -2.28515625,
    -2.751953125,
    -3.3515625,
    -4.3984375,
    -2.78125,
    -4.0703125,
    -4.44140625,
    -4.53125,
    -4.6484375,
    -2.67578125,
    -3.443359375,
    -3.345703125,
    -1.5380859375,
    -4.890625,
    -5.890625,
    -3.833984375,
    -4.38671875,
    -2.9765625,
    -5.05859375,
    -4.1328125,
    -6.58984375,
    -4.1015625,
    -5.28125,
    -5.96875,
    -4.32421875,
    -2.974609375,
    -4.94921875,
    -5.16015625,
    -6.30078125,
    -4.4609375,
    -3.568359375,
    -6.453125,
    -4.36328125,
    -5.2421875,
    -6.09765625,
    -4.171875,
    -4.81640625,
    -4.1015625,
    -4.859375,
    -5.1328125,
    -4.1015625,
    -5.3828125,
    -4.21875,
    -3.337890625,
    -5.4921875,
    -3.205078125,
    -3.3828125,
    -4.14453125,
    -4.8828125,
    -4.7421875,
    -6.640625,
    -7.4609375,
    -7.078125,
    -8.078125,
    -5.90234375,
    -3.873046875,
    -7.3984375,
    -7.72265625,
    -2.56640625,
    -7.51953125,
    -4.25390625,
    -8.0703125,
    -5.32421875,
    -3.361328125,
    -5.58984375,
    -3.68359375,
    -6.42578125,
    -4.26953125,
    -7.1875,
    -3.99609375,
    -6.140625,
    -4.640625,
    -4.17578125,
    -3.0546875,
    -4.6015625,
    -3.6484375,
    -4.58203125,
    -2.58203125,
    -4.19921875,
    -3.228515625,
    -4.84765625,
    -3.748046875,
    -4.328125,
    -4.26171875,
    -3.7265625,
    -4.62109375,
    -4.609375,
    -5.015625,
    -3.537109375,
    -3.134765625,
    -4.23828125,
    -5.75390625,
    -4.95703125,
    -3.392578125,
    -4.05078125,
    -4.5546875,
    -5.82421875,
    -4.11328125,
    -5.31640625,
    -4.46875,
    -6.26171875,
    -4.6171875,
    -4.86328125,
    -5.2265625,
    -6.04296875,
    -5.06640625,
    -3.90625,
    -4.65234375,
    -5.953125,
    -3.810546875,
    -4.0234375,
    -3.5390625,
    -4.48046875,
    -3.234375,
    -3.748046875,
    -2.435546875,
    -5.375,
    -5.7890625,
    -2.857421875,
    -2.970703125,
    -5.80078125,
    -6.1953125,
    -2.849609375,
    -5.91015625,
    -6.48828125,
    -5.015625,
    -4.07421875,
    -3.873046875,
    -4.86328125,
    -6.20703125,
    -2.962890625,
    -3.4140625,
    -4.33203125,
    -3.4921875,
    -2.564453125,
    -5.203125,
    -5.14453125,
    -5.99609375,
    -5.70703125,
    -5.36328125,
    -4.74609375,
    -4.6875,
    -2.744140625,
    -1.166015625,
    -3.513671875,
    -3.04296875,
    -1.603515625,
    -1.8203125,
    -1.8349609375,
    -2.88671875,
    -0.48388671875,
    -1.9755859375,
    -2.2421875,
    -1.6845703125,
    -0.9609375,
    -2.966796875,
    -1.9716796875,
    -2.837890625,
    -2.6015625,
    -3.931640625,
    -2.01953125,
    -2.244140625,
    -2.548828125,
    -2.240234375,
    -3.087890625,
    -0.849609375,
    -2.212890625,
    -3.732421875,
    -3.45703125,
    -2.859375,
    -2.4296875,
    -3.349609375,
    -2.546875,
    -3.228515625,
    -2.123046875,
    -2.568359375,
    -1.8134765625,
    -1.470703125,
    -1.52734375,
    -2.208984375,
    -2.912109375,
    -0.77001953125,
    -1.8447265625,
    -2.541015625,
    -1.533203125,
    -1.8212890625,
    -1.798828125,
    -2.201171875,
    -1.9755859375,
    -2.22265625,
    -1.37109375,
    -1.28515625,
    -2.041015625,
    -1.732421875,
    -1.984375,
    -2.49609375,
    -1.1611328125,
    -2.19921875,
    -2.556640625,
    0.2188720703125,
    -2.0859375,
    -1.513671875,
    -1.0966796875,
    -0.01303863525390625,
    -0.8486328125,
    -2.6640625,
    -0.93115234375,
    -1.677734375,
    -1.984375,
    -1.2548828125,
    -1.9091796875,
    -0.9462890625,
    -1.515625,
    -0.64453125,
    -1.0859375,
    -0.3466796875,
    -0.22412109375,
    -0.54638671875,
    0.9208984375,
    0.833984375,
    -0.30908203125,
    -0.42333984375,
    -0.7314453125,
    0.491943359375,
    -0.70166015625,
    0.369873046875,
    0.35986328125,
    1.208984375,
    -0.296142578125,
    0.21142578125,
    -0.88671875,
    0.303466796875,
    0.75927734375,
    -0.888671875,
    -0.45947265625,
    -0.12139892578125,
    0.953125,
    0.5478515625,
    -0.73046875,
    -0.72705078125,
    2.23828125,
    -0.01377105712890625,
    0.00981903076171875,
    -0.273681640625,
    0.34765625,
    -0.370849609375,
    -4.76953125
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9959732890129089,
    0.003665885655209422,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00036087396438233554
   ],
   "p": [
    0.9963328242301941,
    0.0036672090645879507
   ],
   "p_oracle": [
    0.9963294863700867,
    0.0036705152597278357
   ],
   "abstain": 0.00036087396438233554,
   "abstain_oracle": 0.00036291085416451097,
   "abs_delta_abstain": 2.0368897821754217e-06,
   "oracle_margin": 0.9926589727401733,
   "max_abs_delta_p": 3.337860107421875e-06,
   "mean_abs_delta_p": 3.322027623653412e-06,
   "max_abs_delta_raw_logits": 0.0070743560791015625,
   "mean_abs_delta_raw_logits": 0.0016561710508540273,
   "logits_min": -8.078125,
   "logits_max": 3.20703125,
   "raw_logits_min": -8.078125,
   "raw_logits_max": 3.20703125,
   "raw_logits_absmax": 8.078125,
   "oracle_logits_min": -8.083602905273438,
   "oracle_logits_max": 3.207350730895996,
   "oracle_raw_logits_absmax": 8.083602905273438,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.48141737503465265,
   "seconds": 0.48141737503465265,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r02-action",
   "request_id": "r02",
   "question_id": "action",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 108,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    3.564453125,
    -2.40234375,
    -1.70703125,
    -2.6328125,
    -2.1171875,
    -2.55078125,
    -2.29296875,
    -3.560546875,
    -3.072265625,
    -2.189453125,
    -2.626953125,
    -3.443359375,
    -4.4296875,
    -2.5078125,
    -3.943359375,
    -3.953125,
    -3.986328125,
    -4.7734375,
    -2.712890625,
    -3.08984375,
    -3.181640625,
    -1.8349609375,
    -5.18359375,
    -6.08984375,
    -3.74609375,
    -4.1953125,
    -2.787109375,
    -4.6484375,
    -3.970703125,
    -6.20703125,
    -3.904296875,
    -4.80859375,
    -5.70703125,
    -4.171875,
    -3.005859375,
    -4.5,
    -5.03125,
    -6.0234375,
    -4.203125,
    -3.26953125,
    -6.28515625,
    -4.3125,
    -4.9609375,
    -6.1875,
    -3.869140625,
    -4.6171875,
    -4.1015625,
    -4.55859375,
    -5.109375,
    -4.22265625,
    -5.16796875,
    -4.05859375,
    -3.275390625,
    -5.45703125,
    -3.25390625,
    -3.40625,
    -4.15234375,
    -4.94140625,
    -4.6484375,
    -6.53515625,
    -7.27734375,
    -6.9140625,
    -8.0234375,
    -5.70703125,
    -3.86328125,
    -7.17578125,
    -7.55859375,
    -2.55859375,
    -7.328125,
    -4.07421875,
    -7.87109375,
    -5.21875,
    -3.173828125,
    -5.328125,
    -3.498046875,
    -5.91796875,
    -4.09375,
    -6.8984375,
    -3.796875,
    -5.8984375,
    -4.6328125,
    -4.078125,
    -2.9375,
    -4.46875,
    -3.345703125,
    -4.3984375,
    -2.384765625,
    -4.1484375,
    -3.0234375,
    -4.59765625,
    -3.806640625,
    -4.10546875,
    -4.33984375,
    -3.564453125,
    -4.4609375,
    -4.41796875,
    -4.88671875,
    -3.43359375,
    -2.916015625,
    -4.19921875,
    -5.44921875,
    -4.71484375,
    -3.216796875,
    -3.841796875,
    -4.50390625,
    -5.51171875,
    -3.65625,
    -5.03515625,
    -4.2734375,
    -5.95703125,
    -4.3828125,
    -4.703125,
    -5.078125,
    -5.94921875,
    -4.8203125,
    -3.69140625,
    -4.5234375,
    -5.68359375,
    -3.666015625,
    -3.7890625,
    -3.244140625,
    -4.41796875,
    -3.0390625,
    -3.693359375,
    -2.283203125,
    -5.1015625,
    -5.74609375,
    -2.423828125,
    -2.806640625,
    -5.47265625,
    -5.8671875,
    -2.591796875,
    -5.66796875,
    -6.26171875,
    -4.93359375,
    -3.955078125,
    -3.619140625,
    -4.73828125,
    -6.05078125,
    -2.765625,
    -3.494140625,
    -4.203125,
    -3.244140625,
    -2.29296875,
    -5.0,
    -5.1015625,
    -5.76953125,
    -5.3984375,
    -5.23828125,
    -4.5859375,
    -4.55859375,
    -2.787109375,
    -0.943359375,
    -3.439453125,
    -2.75,
    -1.587890625,
    -1.71875,
    -1.9248046875,
    -2.845703125,
    -0.2069091796875,
    -1.6181640625,
    -2.056640625,
    -1.7060546875,
    -0.99560546875,
    -2.95703125,
    -1.666015625,
    -2.60546875,
    -2.330078125,
    -3.509765625,
    -1.837890625,
    -2.19921875,
    -2.392578125,
    -2.228515625,
    -2.9921875,
    -0.5771484375,
    -1.884765625,
    -3.525390625,
    -3.29296875,
    -2.853515625,
    -2.072265625,
    -3.015625,
    -2.466796875,
    -2.927734375,
    -2.05078125,
    -2.431640625,
    -1.935546875,
    -1.3505859375,
    -1.3935546875,
    -2.1484375,
    -2.71484375,
    -0.517578125,
    -1.6318359375,
    -2.275390625,
    -1.7333984375,
    -1.931640625,
    -1.8408203125,
    -2.005859375,
    -1.70703125,
    -2.0703125,
    -1.115234375,
    -1.1103515625,
    -1.80859375,
    -1.447265625,
    -2.013671875,
    -2.431640625,
    -1.130859375,
    -2.052734375,
    -2.462890625,
    0.2861328125,
    -1.98046875,
    -1.4208984375,
    -0.9267578125,
    0.0931396484375,
    -0.76171875,
    -2.71875,
    -0.71044921875,
    -1.5654296875,
    -1.8125,
    -0.98974609375,
    -1.7392578125,
    -1.03515625,
    -1.8330078125,
    -0.71044921875,
    -1.1416015625,
    -0.321044921875,
    -0.1878662109375,
    -0.38720703125,
    1.140625,
    0.73583984375,
    -0.18359375,
    -0.46533203125,
    -0.69140625,
    0.72607421875,
    -0.728515625,
    0.2327880859375,
    0.56591796875,
    1.138671875,
    -0.14697265625,
    0.11920166015625,
    -0.71435546875,
    0.289794921875,
    0.74169921875,
    -0.82177734375,
    -0.47900390625,
    -0.1656494140625,
    0.8427734375,
    0.74560546875,
    -0.59765625,
    -0.86328125,
    2.29296875,
    -0.10137939453125,
    -0.1182861328125,
    -0.2391357421875,
    0.1744384765625,
    -0.404541015625,
    -4.953125
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9893559813499451,
    0.0035261402372270823,
    0.0068009826354682446,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003167865506839007
   ],
   "p": [
    0.9896695613861084,
    0.003527257824316621,
    0.006803138181567192
   ],
   "p_oracle": [
    0.989686131477356,
    0.0035125669091939926,
    0.006801280193030834
   ],
   "abstain": 0.0003167865506839007,
   "abstain_oracle": 0.00031577961635775864,
   "abs_delta_abstain": 1.0069343261420727e-06,
   "oracle_margin": 0.9828848242759705,
   "max_abs_delta_p": 1.6570091247558594e-05,
   "mean_abs_delta_p": 1.1039664968848228e-05,
   "max_abs_delta_raw_logits": 0.006712436676025391,
   "mean_abs_delta_raw_logits": 0.0012005677563138306,
   "logits_min": -8.0234375,
   "logits_max": 3.564453125,
   "raw_logits_min": -8.0234375,
   "raw_logits_max": 3.564453125,
   "raw_logits_absmax": 8.0234375,
   "oracle_logits_min": -8.02131175994873,
   "oracle_logits_max": 3.5675013065338135,
   "oracle_raw_logits_absmax": 8.02131175994873,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5971651659347117,
   "seconds": 0.5971651659347117,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r02-lamp",
   "request_id": "r02",
   "question_id": "lamp",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 79,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.27734375,
    -2.916015625,
    3.17578125,
    -1.595703125,
    -2.123046875,
    -2.337890625,
    -2.365234375,
    -2.80859375,
    -2.998046875,
    -1.7900390625,
    -2.84375,
    -2.62109375,
    -4.1640625,
    -2.560546875,
    -3.517578125,
    -4.1796875,
    -3.755859375,
    -4.46484375,
    -2.552734375,
    -2.48828125,
    -4.046875,
    -1.37890625,
    -5.3359375,
    -5.43359375,
    -3.26171875,
    -4.3515625,
    -3.302734375,
    -3.83203125,
    -3.697265625,
    -5.79296875,
    -4.35546875,
    -4.9453125,
    -4.9453125,
    -4.1171875,
    -2.734375,
    -4.8203125,
    -5.265625,
    -6.55859375,
    -4.6640625,
    -3.349609375,
    -5.94140625,
    -3.970703125,
    -4.78125,
    -6.171875,
    -4.0546875,
    -4.7734375,
    -4.296875,
    -4.3515625,
    -4.87109375,
    -4.796875,
    -5.18359375,
    -4.79296875,
    -3.048828125,
    -5.0390625,
    -3.013671875,
    -3.556640625,
    -4.02734375,
    -5.2265625,
    -5.1171875,
    -6.12890625,
    -7.32421875,
    -6.703125,
    -7.50390625,
    -5.88671875,
    -3.365234375,
    -7.0859375,
    -7.37109375,
    -2.84375,
    -7.6171875,
    -4.49609375,
    -7.84765625,
    -5.15625,
    -2.693359375,
    -5.21484375,
    -3.015625,
    -6.12109375,
    -3.873046875,
    -6.55078125,
    -4.140625,
    -5.5078125,
    -4.546875,
    -3.783203125,
    -2.552734375,
    -4.5703125,
    -3.27734375,
    -4.59375,
    -2.576171875,
    -4.14453125,
    -3.0859375,
    -4.31640625,
    -3.4140625,
    -3.7578125,
    -4.22265625,
    -3.126953125,
    -4.828125,
    -4.5703125,
    -4.94140625,
    -3.30859375,
    -3.072265625,
    -4.48828125,
    -5.48046875,
    -4.42578125,
    -3.58984375,
    -3.6015625,
    -4.33984375,
    -5.36328125,
    -3.134765625,
    -5.12890625,
    -4.0390625,
    -5.7734375,
    -3.88671875,
    -4.4609375,
    -4.91796875,
    -5.546875,
    -4.3671875,
    -3.541015625,
    -4.31640625,
    -5.62109375,
    -3.759765625,
    -3.115234375,
    -3.48046875,
    -4.296875,
    -2.890625,
    -3.169921875,
    -2.43359375,
    -5.0625,
    -5.1953125,
    -2.939453125,
    -2.63671875,
    -5.35546875,
    -5.3203125,
    -2.4296875,
    -5.50390625,
    -6.171875,
    -5.43359375,
    -4.76953125,
    -3.6328125,
    -4.3046875,
    -5.6328125,
    -2.998046875,
    -2.9609375,
    -3.98046875,
    -3.462890625,
    -2.677734375,
    -4.05078125,
    -4.703125,
    -5.18359375,
    -5.0625,
    -5.03125,
    -4.27734375,
    -4.51953125,
    -2.775390625,
    -0.548828125,
    -3.484375,
    -2.251953125,
    -1.1572265625,
    -1.37109375,
    -1.6787109375,
    -2.75390625,
    -0.1619873046875,
    -1.673828125,
    -2.005859375,
    -1.19921875,
    -1.25,
    -3.1640625,
    -2.154296875,
    -2.994140625,
    -1.91796875,
    -3.578125,
    -1.7666015625,
    -2.087890625,
    -1.962890625,
    -1.7744140625,
    -2.5,
    -0.51513671875,
    -1.5751953125,
    -2.91796875,
    -3.13671875,
    -2.501953125,
    -2.607421875,
    -2.751953125,
    -2.26171875,
    -2.8828125,
    -2.08984375,
    -2.912109375,
    -1.6279296875,
    -2.193359375,
    -1.283203125,
    -1.4453125,
    -2.55078125,
    -0.6181640625,
    -1.51171875,
    -2.431640625,
    -1.525390625,
    -1.8994140625,
    -2.314453125,
    -1.64453125,
    -1.4892578125,
    -1.6884765625,
    -1.1025390625,
    -0.8359375,
    -1.923828125,
    -1.673828125,
    -1.771484375,
    -2.517578125,
    -0.76123046875,
    -2.142578125,
    -2.32421875,
    -0.036102294921875,
    -1.7705078125,
    -0.85986328125,
    -0.84326171875,
    0.00949859619140625,
    -1.361328125,
    -2.990234375,
    -0.7939453125,
    -1.4091796875,
    -2.056640625,
    -1.189453125,
    -1.349609375,
    -0.82177734375,
    -1.806640625,
    -1.177734375,
    -1.390625,
    -0.03887939453125,
    -0.180908203125,
    -0.1939697265625,
    1.162109375,
    1.2197265625,
    -0.30224609375,
    -0.273681640625,
    -0.541015625,
    -0.0714111328125,
    -1.0322265625,
    0.7666015625,
    0.8203125,
    1.2470703125,
    -0.493896484375,
    -0.30078125,
    -0.888671875,
    0.515625,
    0.497314453125,
    -0.7724609375,
    0.0035533905029296875,
    -0.03271484375,
    1.0302734375,
    0.469482421875,
    -0.759765625,
    -0.54150390625,
    1.9130859375,
    -0.345458984375,
    -0.6962890625,
    -0.2086181640625,
    0.436279296875,
    -0.07666015625,
    -6.2421875
   ],
   "raw256_argmax": 2,
   "option_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "p_full": [
    0.005738045554608107,
    0.003138558706268668,
    0.9909878373146057,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00013554100587498397
   ],
   "p": [
    0.0057388232089579105,
    0.003138984087854624,
    0.9911221861839294
   ],
   "p_oracle": [
    0.0057459608651697636,
    0.003123035654425621,
    0.9911310076713562
   ],
   "abstain": 0.00013554100587498397,
   "abstain_oracle": 0.0001346878707408905,
   "abs_delta_abstain": 8.531351340934634e-07,
   "oracle_margin": 0.9853850603103638,
   "max_abs_delta_p": 1.5948433429002762e-05,
   "mean_abs_delta_p": 1.0635859022537867e-05,
   "max_abs_delta_raw_logits": 0.0065765380859375,
   "mean_abs_delta_raw_logits": 0.0010169922898057848,
   "logits_min": -7.84765625,
   "logits_max": 3.17578125,
   "raw_logits_min": -7.84765625,
   "raw_logits_max": 3.17578125,
   "raw_logits_absmax": 7.84765625,
   "oracle_logits_min": -7.847484588623047,
   "oracle_logits_max": 3.175899028778076,
   "oracle_raw_logits_absmax": 7.847484588623047,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.43884737498592585,
   "seconds": 0.43884737498592585,
   "full_vocab_argmax_id": 2,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r02-cold",
   "request_id": "r02",
   "question_id": "cold",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 70,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -1.775390625,
    0.26171875,
    -1.076171875,
    -1.375,
    -1.15234375,
    -2.87890625,
    -1.9833984375,
    -2.765625,
    -3.572265625,
    -1.6923828125,
    -2.13671875,
    -3.44140625,
    -4.1171875,
    -2.5625,
    -2.888671875,
    -3.86328125,
    -3.638671875,
    -4.97265625,
    -2.837890625,
    -2.6015625,
    -3.365234375,
    -1.7353515625,
    -4.88671875,
    -5.91796875,
    -3.48828125,
    -4.42578125,
    -2.451171875,
    -4.8203125,
    -3.494140625,
    -5.12109375,
    -4.00390625,
    -3.890625,
    -5.44921875,
    -3.9375,
    -2.462890625,
    -4.42578125,
    -4.87890625,
    -5.54296875,
    -4.375,
    -2.65234375,
    -6.54296875,
    -3.93359375,
    -4.4921875,
    -5.625,
    -3.1640625,
    -4.28125,
    -4.0078125,
    -4.31640625,
    -4.4453125,
    -4.1484375,
    -5.0,
    -4.05859375,
    -2.638671875,
    -4.75390625,
    -2.4765625,
    -3.3359375,
    -3.5078125,
    -4.390625,
    -4.00390625,
    -5.21875,
    -6.92578125,
    -6.01953125,
    -7.52734375,
    -5.71875,
    -3.171875,
    -7.17578125,
    -6.97265625,
    -2.10546875,
    -7.25,
    -4.0703125,
    -7.375,
    -4.83203125,
    -2.154296875,
    -5.3203125,
    -3.578125,
    -5.58203125,
    -4.1640625,
    -6.109375,
    -3.5390625,
    -5.36328125,
    -3.8828125,
    -3.41796875,
    -2.388671875,
    -3.876953125,
    -3.130859375,
    -3.69140625,
    -2.4296875,
    -3.181640625,
    -2.875,
    -3.962890625,
    -3.12109375,
    -3.619140625,
    -3.162109375,
    -3.89453125,
    -4.25,
    -4.47265625,
    -4.45703125,
    -3.2421875,
    -2.896484375,
    -3.9453125,
    -5.01171875,
    -4.68359375,
    -2.763671875,
    -3.423828125,
    -4.09375,
    -5.26171875,
    -3.021484375,
    -4.5625,
    -3.74609375,
    -4.96484375,
    -4.0546875,
    -4.19921875,
    -4.3203125,
    -5.58984375,
    -4.16015625,
    -3.2265625,
    -4.5,
    -5.05859375,
    -3.83984375,
    -2.966796875,
    -2.9921875,
    -4.05078125,
    -2.890625,
    -3.3046875,
    -2.16015625,
    -4.91796875,
    -4.8046875,
    -2.544921875,
    -2.68359375,
    -4.6171875,
    -5.11328125,
    -2.5625,
    -5.109375,
    -5.6953125,
    -4.9921875,
    -4.14453125,
    -3.35546875,
    -4.2734375,
    -5.3125,
    -2.052734375,
    -3.134765625,
    -3.775390625,
    -3.146484375,
    -3.005859375,
    -4.1484375,
    -4.62890625,
    -5.328125,
    -4.7734375,
    -4.78515625,
    -3.990234375,
    -4.1328125,
    -2.66796875,
    -0.8125,
    -2.59375,
    -1.99609375,
    -1.0595703125,
    -1.4091796875,
    -1.5703125,
    -2.8359375,
    -0.54736328125,
    -1.70703125,
    -1.876953125,
    -0.8857421875,
    -1.0556640625,
    -2.8046875,
    -2.111328125,
    -2.181640625,
    -1.623046875,
    -3.322265625,
    -1.9052734375,
    -1.7568359375,
    -1.953125,
    -1.6318359375,
    -2.6796875,
    -0.7412109375,
    -1.4736328125,
    -2.943359375,
    -3.01171875,
    -2.513671875,
    -2.673828125,
    -2.826171875,
    -1.95703125,
    -2.30078125,
    -1.66796875,
    -2.00390625,
    -1.41796875,
    -1.3046875,
    -1.283203125,
    -1.6123046875,
    -1.6337890625,
    -0.464599609375,
    -1.455078125,
    -2.203125,
    -1.0498046875,
    -1.4150390625,
    -1.6494140625,
    -1.7353515625,
    -1.1259765625,
    -1.5400390625,
    -0.9306640625,
    -0.9658203125,
    -1.4072265625,
    -1.23828125,
    -1.4091796875,
    -1.8837890625,
    -0.49853515625,
    -2.267578125,
    -2.482421875,
    -0.01299285888671875,
    -1.556640625,
    -0.424560546875,
    -0.77099609375,
    -0.1553955078125,
    -0.81982421875,
    -2.32421875,
    -0.7177734375,
    -1.27734375,
    -2.103515625,
    -1.298828125,
    -1.3388671875,
    -1.3212890625,
    -1.6337890625,
    -1.369140625,
    -1.01953125,
    -0.1260986328125,
    0.359130859375,
    -0.1866455078125,
    0.9541015625,
    1.0810546875,
    -0.415771484375,
    -0.494873046875,
    -0.312255859375,
    -0.2333984375,
    -0.7978515625,
    0.484130859375,
    1.158203125,
    1.4697265625,
    -0.56689453125,
    -0.257568359375,
    -0.587890625,
    0.546875,
    0.70947265625,
    -0.4912109375,
    0.08935546875,
    0.00934600830078125,
    0.279296875,
    0.406494140625,
    -0.6318359375,
    -0.8115234375,
    1.6337890625,
    -0.167724609375,
    -0.81640625,
    -0.17431640625,
    0.50537109375,
    -0.71533203125,
    -8.421875
   ],
   "raw256_argmax": 249,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.1167476624250412,
    0.8830938339233398,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00015853281365707517
   ],
   "p": [
    0.11676616966724396,
    0.8832338452339172
   ],
   "p_oracle": [
    0.11606618016958237,
    0.8839338421821594
   ],
   "abstain": 0.00015853281365707517,
   "abstain_oracle": 0.0001577989023644477,
   "abs_delta_abstain": 7.339112926274538e-07,
   "oracle_margin": 0.7678676843643188,
   "max_abs_delta_p": 0.0006999969482421875,
   "mean_abs_delta_p": 0.000699993222951889,
   "max_abs_delta_raw_logits": 0.004258155822753906,
   "mean_abs_delta_raw_logits": 0.0012416931858751923,
   "logits_min": -8.421875,
   "logits_max": 1.6337890625,
   "raw_logits_min": -8.421875,
   "raw_logits_max": 1.6337890625,
   "raw_logits_absmax": 8.421875,
   "oracle_logits_min": -8.42408561706543,
   "oracle_logits_max": 1.6328606605529785,
   "oracle_raw_logits_absmax": 8.42408561706543,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3891596249304712,
   "seconds": 0.3891596249304712,
   "full_vocab_argmax_id": 249,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r03-color",
   "request_id": "r03",
   "question_id": "color",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 85,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.28125,
    3.55078125,
    -2.689453125,
    -2.66015625,
    -2.107421875,
    -2.861328125,
    -2.8984375,
    -3.849609375,
    -3.9453125,
    -2.734375,
    -3.361328125,
    -4.09375,
    -5.3984375,
    -3.49609375,
    -4.48828125,
    -4.828125,
    -4.0859375,
    -4.26171875,
    -3.06640625,
    -3.103515625,
    -3.37109375,
    -2.048828125,
    -5.171875,
    -6.99609375,
    -3.919921875,
    -5.31640625,
    -2.51171875,
    -5.515625,
    -4.5078125,
    -7.0078125,
    -4.609375,
    -4.9375,
    -6.29296875,
    -5.3046875,
    -3.3203125,
    -5.54296875,
    -5.3984375,
    -6.01953125,
    -4.2109375,
    -3.1953125,
    -7.16796875,
    -4.41796875,
    -4.84375,
    -6.1640625,
    -4.6796875,
    -5.19921875,
    -4.6015625,
    -4.52734375,
    -5.25,
    -4.63671875,
    -5.828125,
    -4.8515625,
    -2.69921875,
    -4.98828125,
    -2.822265625,
    -3.275390625,
    -3.693359375,
    -4.78515625,
    -4.97265625,
    -5.87109375,
    -7.69921875,
    -6.4453125,
    -7.7109375,
    -6.69921875,
    -3.625,
    -7.546875,
    -7.6875,
    -2.455078125,
    -7.85546875,
    -4.08203125,
    -8.046875,
    -5.46875,
    -2.75390625,
    -6.2265625,
    -3.8671875,
    -6.328125,
    -4.296875,
    -6.953125,
    -4.2734375,
    -5.78125,
    -4.3828125,
    -4.2734375,
    -2.767578125,
    -4.22265625,
    -3.560546875,
    -4.0703125,
    -2.619140625,
    -3.82421875,
    -3.115234375,
    -4.9765625,
    -3.537109375,
    -4.20703125,
    -4.3671875,
    -4.1328125,
    -4.625,
    -4.85546875,
    -5.31640625,
    -4.0859375,
    -4.3359375,
    -4.2421875,
    -6.23828125,
    -4.54296875,
    -3.697265625,
    -4.24609375,
    -4.62890625,
    -6.234375,
    -4.4609375,
    -5.8515625,
    -4.3125,
    -5.65234375,
    -4.48828125,
    -4.84765625,
    -5.44140625,
    -6.6953125,
    -4.890625,
    -4.3359375,
    -4.64453125,
    -6.08984375,
    -4.40625,
    -3.513671875,
    -3.490234375,
    -4.078125,
    -3.326171875,
    -3.9609375,
    -2.58203125,
    -6.15234375,
    -5.8984375,
    -3.392578125,
    -3.287109375,
    -5.67578125,
    -5.7421875,
    -3.873046875,
    -5.75390625,
    -5.98046875,
    -4.9765625,
    -4.74609375,
    -4.18359375,
    -5.1640625,
    -5.91015625,
    -3.275390625,
    -3.6875,
    -4.2578125,
    -3.927734375,
    -3.244140625,
    -5.0234375,
    -5.56640625,
    -6.203125,
    -5.2578125,
    -4.8671875,
    -4.6875,
    -5.0,
    -3.365234375,
    -1.220703125,
    -3.546875,
    -2.669921875,
    -1.81640625,
    -2.05859375,
    -2.130859375,
    -3.55859375,
    -0.420654296875,
    -2.279296875,
    -2.1796875,
    -1.947265625,
    -1.298828125,
    -2.630859375,
    -2.4375,
    -3.013671875,
    -1.9873046875,
    -3.54296875,
    -2.0234375,
    -2.455078125,
    -2.623046875,
    -2.666015625,
    -3.181640625,
    -1.017578125,
    -2.06640625,
    -3.466796875,
    -3.466796875,
    -3.193359375,
    -3.541015625,
    -3.552734375,
    -3.435546875,
    -2.666015625,
    -2.146484375,
    -2.66015625,
    -1.625,
    -1.2138671875,
    -1.814453125,
    -1.953125,
    -2.275390625,
    -1.052734375,
    -1.8349609375,
    -3.025390625,
    -1.9189453125,
    -1.5078125,
    -2.244140625,
    -2.0390625,
    -2.013671875,
    -1.7587890625,
    -1.478515625,
    -1.27734375,
    -1.94140625,
    -1.984375,
    -1.955078125,
    -2.533203125,
    -1.251953125,
    -2.857421875,
    -2.58203125,
    -0.457763671875,
    -2.447265625,
    -0.92919921875,
    -1.4287109375,
    -0.9482421875,
    -1.701171875,
    -2.529296875,
    -1.3623046875,
    -1.994140625,
    -2.4609375,
    -1.826171875,
    -2.0390625,
    -1.3212890625,
    -1.669921875,
    -1.298828125,
    -1.3876953125,
    -0.1356201171875,
    0.06982421875,
    -0.333984375,
    0.8896484375,
    1.0712890625,
    -0.0132598876953125,
    -0.859375,
    -0.62890625,
    0.1910400390625,
    -0.77099609375,
    0.546875,
    1.103515625,
    1.451171875,
    -0.66455078125,
    0.0521240234375,
    -0.8251953125,
    0.271728515625,
    0.83154296875,
    -0.68408203125,
    -0.1094970703125,
    0.099365234375,
    0.62841796875,
    0.453369140625,
    -0.2235107421875,
    -0.97509765625,
    1.509765625,
    -0.082275390625,
    -0.70751953125,
    -0.154052734375,
    0.473876953125,
    -0.43310546875,
    -4.140625
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0040178485214710236,
    0.9925563931465149,
    0.0027322242967784405,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0006936343270353973
   ],
   "p": [
    0.004020636901259422,
    0.9932452440261841,
    0.0027341204695403576
   ],
   "p_oracle": [
    0.004039411433041096,
    0.993238091468811,
    0.0027225520461797714
   ],
   "abstain": 0.0006936343270353973,
   "abstain_oracle": 0.0006879517459310591,
   "abs_delta_abstain": 5.682581104338169e-06,
   "oracle_margin": 0.9891986846923828,
   "max_abs_delta_p": 1.877453178167343e-05,
   "mean_abs_delta_p": 1.2498504171768824e-05,
   "max_abs_delta_raw_logits": 0.013645172119140625,
   "mean_abs_delta_raw_logits": 0.0038388704124372452,
   "logits_min": -8.046875,
   "logits_max": 3.55078125,
   "raw_logits_min": -8.046875,
   "raw_logits_max": 3.55078125,
   "raw_logits_absmax": 8.046875,
   "oracle_logits_min": -8.06052017211914,
   "oracle_logits_max": 3.5530474185943604,
   "oracle_raw_logits_absmax": 8.06052017211914,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4707033750601113,
   "seconds": 0.4707033750601113,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r03-shape",
   "request_id": "r03",
   "question_id": "shape",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 124,
   "nopts": 12,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.0615234375,
    -1.42578125,
    -2.03125,
    -3.787109375,
    4.8515625,
    -1.7744140625,
    -1.904296875,
    -3.50390625,
    -3.955078125,
    -2.02734375,
    -3.294921875,
    -2.43359375,
    -3.9296875,
    -3.171875,
    -4.2890625,
    -4.68359375,
    -3.9296875,
    -5.09765625,
    -2.470703125,
    -2.78125,
    -3.548828125,
    -2.103515625,
    -4.734375,
    -6.70703125,
    -3.97265625,
    -4.1953125,
    -2.85546875,
    -5.04296875,
    -3.697265625,
    -6.1484375,
    -4.1484375,
    -4.44921875,
    -6.3125,
    -4.51953125,
    -2.412109375,
    -4.38671875,
    -5.3359375,
    -6.13671875,
    -4.01953125,
    -2.810546875,
    -7.171875,
    -3.970703125,
    -4.9609375,
    -5.91015625,
    -5.1328125,
    -5.22265625,
    -4.76953125,
    -4.25,
    -4.97265625,
    -4.203125,
    -5.234375,
    -4.24609375,
    -3.078125,
    -5.484375,
    -2.822265625,
    -2.990234375,
    -4.18359375,
    -5.67578125,
    -4.984375,
    -6.48046875,
    -7.5,
    -7.00390625,
    -7.6953125,
    -6.50390625,
    -3.564453125,
    -7.66796875,
    -7.59375,
    -2.88671875,
    -7.78515625,
    -4.37109375,
    -7.6953125,
    -5.13671875,
    -2.666015625,
    -5.4140625,
    -3.6171875,
    -5.90625,
    -4.5625,
    -7.12890625,
    -3.720703125,
    -5.58203125,
    -4.71875,
    -4.58984375,
    -2.923828125,
    -4.296875,
    -3.279296875,
    -4.4453125,
    -2.494140625,
    -3.86328125,
    -3.12109375,
    -4.546875,
    -3.697265625,
    -4.60546875,
    -4.30078125,
    -4.40625,
    -5.23046875,
    -5.0546875,
    -4.9765625,
    -3.287109375,
    -3.810546875,
    -4.44921875,
    -5.41796875,
    -4.48046875,
    -3.123046875,
    -3.5234375,
    -4.70703125,
    -5.2890625,
    -4.07421875,
    -5.36328125,
    -4.62890625,
    -6.1640625,
    -3.95703125,
    -4.94140625,
    -4.61328125,
    -6.46484375,
    -4.19921875,
    -3.615234375,
    -4.91796875,
    -5.90234375,
    -4.0390625,
    -3.287109375,
    -3.49609375,
    -4.07421875,
    -2.96484375,
    -3.76171875,
    -2.23046875,
    -5.9453125,
    -5.4375,
    -2.546875,
    -3.03125,
    -4.921875,
    -6.171875,
    -3.087890625,
    -5.7421875,
    -6.64453125,
    -5.21484375,
    -4.27734375,
    -3.962890625,
    -4.39453125,
    -5.453125,
    -2.419921875,
    -3.142578125,
    -3.970703125,
    -3.470703125,
    -2.498046875,
    -4.38671875,
    -5.53125,
    -5.7578125,
    -5.6953125,
    -5.375,
    -5.06640625,
    -4.8671875,
    -3.76171875,
    -0.391357421875,
    -3.484375,
    -2.904296875,
    -1.458984375,
    -1.3974609375,
    -1.849609375,
    -2.943359375,
    0.27978515625,
    -1.8369140625,
    -2.29296875,
    -1.880859375,
    -1.373046875,
    -2.736328125,
    -2.359375,
    -2.48046875,
    -2.7734375,
    -3.109375,
    -2.087890625,
    -2.51171875,
    -2.615234375,
    -2.12109375,
    -3.34765625,
    -1.5751953125,
    -1.5576171875,
    -2.888671875,
    -3.650390625,
    -3.083984375,
    -2.24609375,
    -2.890625,
    -2.66015625,
    -2.91015625,
    -1.6796875,
    -2.94921875,
    -1.8359375,
    -1.9814453125,
    -1.94140625,
    -2.0078125,
    -2.501953125,
    -0.70556640625,
    -1.701171875,
    -1.9599609375,
    -1.9697265625,
    -2.041015625,
    -1.7900390625,
    -1.8349609375,
    -1.8603515625,
    -1.9716796875,
    -1.12109375,
    -1.083984375,
    -1.58203125,
    -1.9609375,
    -2.126953125,
    -2.35546875,
    -0.935546875,
    -2.427734375,
    -2.1484375,
    -0.1640625,
    -1.978515625,
    -0.59375,
    -0.97802734375,
    -0.82470703125,
    -1.275390625,
    -2.98046875,
    -0.9482421875,
    -2.20703125,
    -1.908203125,
    -1.1142578125,
    -1.154296875,
    -0.73583984375,
    -1.984375,
    -0.974609375,
    -1.2861328125,
    -0.336181640625,
    -0.08087158203125,
    0.0980224609375,
    1.1943359375,
    1.7763671875,
    -0.021240234375,
    -0.6669921875,
    -0.6943359375,
    0.034820556640625,
    -0.76806640625,
    0.81591796875,
    0.712890625,
    1.5703125,
    -0.740234375,
    -0.399658203125,
    -0.80078125,
    0.60107421875,
    0.5986328125,
    -1.248046875,
    -0.3876953125,
    -0.23974609375,
    0.9404296875,
    0.5908203125,
    -0.89599609375,
    -1.1103515625,
    1.6181640625,
    -0.311767578125,
    -0.3076171875,
    -0.36474609375,
    0.5859375,
    -0.2015380859375,
    -5.93359375
   ],
   "raw256_argmax": 4,
   "option_argmax": 4,
   "oracle_argmax": 4,
   "argmax_agrees": true,
   "p_full": [
    0.0036925843451172113,
    0.0026174767408519983,
    0.0014773097354918718,
    0.0002812413149513304,
    0.9847976565361023,
    0.0018829817418009043,
    0.0016655507497489452,
    0.0003675118205137551,
    0.00023997439711820334,
    0.0014827721752226353,
    0.00044772704131901264,
    0.0010101791704073548,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    3.701869718497619e-05
   ],
   "p": [
    0.00369272124953568,
    0.002617573831230402,
    0.0014773644506931305,
    0.00028125173412263393,
    0.9848341941833496,
    0.0018830515909940004,
    0.0016656124498695135,
    0.00036752544110640883,
    0.00023998328833840787,
    0.0014828271232545376,
    0.000447743630502373,
    0.001010216656140983
   ],
   "p_oracle": [
    0.0036773886531591415,
    0.0026007944252341986,
    0.0014550749910995364,
    0.0002769312704913318,
    0.9849311113357544,
    0.0018732596654444933,
    0.0016641964903101325,
    0.00036053467192687094,
    0.00023951621551532298,
    0.001478387275710702,
    0.0004451809509191662,
    0.0009976954897865653
   ],
   "abstain": 3.701869718497619e-05,
   "abstain_oracle": 3.644650860223919e-05,
   "abs_delta_abstain": 5.721885827369988e-07,
   "oracle_margin": 0.9812537431716919,
   "max_abs_delta_p": 9.691715240478516e-05,
   "mean_abs_delta_p": 1.6152374882949516e-05,
   "max_abs_delta_raw_logits": 0.012567520141601562,
   "mean_abs_delta_raw_logits": 0.0024059087445493788,
   "logits_min": -7.78515625,
   "logits_max": 4.8515625,
   "raw_logits_min": -7.78515625,
   "raw_logits_max": 4.8515625,
   "raw_logits_absmax": 7.78515625,
   "oracle_logits_min": -7.787149429321289,
   "oracle_logits_max": 4.8594279289245605,
   "oracle_raw_logits_absmax": 7.787149429321289,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.6880140830762684,
   "seconds": 0.6880140830762684,
   "full_vocab_argmax_id": 4,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r03-wet",
   "request_id": "r03",
   "question_id": "wet",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 79,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    2.66015625,
    -1.4443359375,
    -1.62890625,
    -2.48828125,
    -2.3671875,
    -2.7890625,
    -2.76171875,
    -3.69921875,
    -2.9921875,
    -1.9677734375,
    -2.318359375,
    -3.318359375,
    -3.705078125,
    -2.263671875,
    -3.37109375,
    -4.1171875,
    -4.0078125,
    -4.796875,
    -2.7421875,
    -3.23046875,
    -3.361328125,
    -1.6513671875,
    -4.984375,
    -5.9375,
    -3.228515625,
    -4.0078125,
    -2.50390625,
    -4.53125,
    -3.751953125,
    -6.08984375,
    -3.783203125,
    -4.75,
    -5.64453125,
    -4.12890625,
    -2.779296875,
    -4.72265625,
    -4.83203125,
    -5.90625,
    -4.40625,
    -3.0390625,
    -6.43359375,
    -4.25,
    -4.84765625,
    -5.76171875,
    -3.91796875,
    -4.265625,
    -3.869140625,
    -4.51953125,
    -4.92578125,
    -4.00390625,
    -5.15234375,
    -4.171875,
    -3.21875,
    -5.11328125,
    -3.05078125,
    -3.353515625,
    -3.888671875,
    -4.484375,
    -4.41015625,
    -6.02734375,
    -7.0234375,
    -6.7109375,
    -7.7734375,
    -5.55078125,
    -3.64453125,
    -6.9765625,
    -7.17578125,
    -2.353515625,
    -7.171875,
    -4.0703125,
    -7.546875,
    -4.8203125,
    -2.873046875,
    -5.2578125,
    -3.45703125,
    -5.87890625,
    -4.01171875,
    -6.60546875,
    -3.673828125,
    -5.63671875,
    -4.3359375,
    -3.677734375,
    -2.658203125,
    -4.28515625,
    -3.150390625,
    -4.23046875,
    -2.220703125,
    -3.84375,
    -3.05859375,
    -4.390625,
    -3.388671875,
    -3.990234375,
    -3.890625,
    -3.38671875,
    -4.375,
    -4.1484375,
    -4.6875,
    -3.2890625,
    -2.826171875,
    -4.02734375,
    -5.125,
    -4.71484375,
    -3.15234375,
    -3.6328125,
    -4.20703125,
    -5.5,
    -3.58984375,
    -4.9296875,
    -4.0390625,
    -5.609375,
    -4.33203125,
    -4.54296875,
    -4.83984375,
    -5.66796875,
    -4.6484375,
    -3.71875,
    -4.578125,
    -5.4140625,
    -3.685546875,
    -3.462890625,
    -3.087890625,
    -4.21875,
    -3.01953125,
    -3.392578125,
    -2.228515625,
    -5.0703125,
    -5.37890625,
    -2.533203125,
    -2.697265625,
    -5.2109375,
    -5.5625,
    -2.45703125,
    -5.26171875,
    -6.0,
    -4.859375,
    -3.947265625,
    -3.46484375,
    -4.703125,
    -5.53515625,
    -2.826171875,
    -3.283203125,
    -4.09375,
    -3.203125,
    -2.521484375,
    -4.70703125,
    -4.7734375,
    -5.41796875,
    -5.078125,
    -5.08203125,
    -4.265625,
    -4.265625,
    -2.572265625,
    -1.009765625,
    -3.240234375,
    -2.689453125,
    -1.2646484375,
    -1.7841796875,
    -1.83203125,
    -2.830078125,
    -0.6083984375,
    -1.7158203125,
    -2.044921875,
    -1.5810546875,
    -0.990234375,
    -2.833984375,
    -1.724609375,
    -2.52734375,
    -2.138671875,
    -3.693359375,
    -1.830078125,
    -1.984375,
    -2.4296875,
    -2.01953125,
    -2.900390625,
    -0.64697265625,
    -1.9892578125,
    -3.451171875,
    -3.236328125,
    -2.7109375,
    -2.203125,
    -3.083984375,
    -2.23046875,
    -2.724609375,
    -1.990234375,
    -2.25390625,
    -1.7734375,
    -1.443359375,
    -1.2373046875,
    -2.154296875,
    -2.482421875,
    -0.388916015625,
    -1.5986328125,
    -2.4140625,
    -1.4541015625,
    -1.4052734375,
    -1.7236328125,
    -2.099609375,
    -1.6083984375,
    -1.873046875,
    -0.99853515625,
    -1.083984375,
    -1.71484375,
    -1.6083984375,
    -1.6689453125,
    -2.369140625,
    -0.94970703125,
    -2.193359375,
    -2.330078125,
    0.066162109375,
    -1.8544921875,
    -1.2666015625,
    -1.033203125,
    0.0316162109375,
    -0.58740234375,
    -2.537109375,
    -0.89892578125,
    -1.5126953125,
    -1.896484375,
    -1.1220703125,
    -1.7958984375,
    -0.9970703125,
    -1.748046875,
    -0.8388671875,
    -1.0869140625,
    -0.2056884765625,
    -0.07391357421875,
    -0.61083984375,
    0.97900390625,
    0.8798828125,
    -0.124267578125,
    -0.2822265625,
    -0.60205078125,
    0.41943359375,
    -0.8330078125,
    0.470947265625,
    0.65087890625,
    1.4140625,
    -0.310546875,
    0.08929443359375,
    -0.83984375,
    0.341552734375,
    0.58984375,
    -0.6953125,
    -0.273193359375,
    -0.1854248046875,
    0.63818359375,
    0.56982421875,
    -0.63525390625,
    -0.95703125,
    2.201171875,
    -0.302978515625,
    -0.317138671875,
    -0.2440185546875,
    0.42919921875,
    -0.49560546875,
    -5.55859375
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9830475449562073,
    0.016672391444444656,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00028003266197629273
   ],
   "p": [
    0.9833229780197144,
    0.016677062958478928
   ],
   "p_oracle": [
    0.9831133484840393,
    0.016886629164218903
   ],
   "abstain": 0.00028003266197629273,
   "abstain_oracle": 0.0002791435399558395,
   "abs_delta_abstain": 8.891220204532146e-07,
   "oracle_margin": 0.9662266969680786,
   "max_abs_delta_p": 0.00020962953567504883,
   "mean_abs_delta_p": 0.0002095978707075119,
   "max_abs_delta_raw_logits": 0.008964776992797852,
   "mean_abs_delta_raw_logits": 0.0015670257853344083,
   "logits_min": -7.7734375,
   "logits_max": 2.66015625,
   "raw_logits_min": -7.7734375,
   "raw_logits_max": 2.66015625,
   "raw_logits_absmax": 7.7734375,
   "oracle_logits_min": -7.771839141845703,
   "oracle_logits_max": 2.6563339233398438,
   "oracle_raw_logits_absmax": 7.771839141845703,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.44007695792242885,
   "seconds": 0.44007695792242885,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r04-status",
   "request_id": "r04",
   "question_id": "status",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 106,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    3.259765625,
    -1.54296875,
    -2.814453125,
    -2.650390625,
    -2.3671875,
    -2.5625,
    -2.52734375,
    -3.869140625,
    -3.64453125,
    -2.453125,
    -2.62890625,
    -3.1796875,
    -4.36328125,
    -2.419921875,
    -3.44140625,
    -3.857421875,
    -3.9609375,
    -4.81640625,
    -3.013671875,
    -3.369140625,
    -3.33203125,
    -1.8447265625,
    -5.15625,
    -5.921875,
    -3.68359375,
    -4.2734375,
    -2.677734375,
    -4.6953125,
    -3.849609375,
    -6.1875,
    -4.05859375,
    -4.46875,
    -5.7421875,
    -4.0625,
    -3.080078125,
    -4.5390625,
    -4.98046875,
    -5.97265625,
    -4.015625,
    -3.0859375,
    -6.1640625,
    -4.30859375,
    -5.08984375,
    -5.6796875,
    -3.65234375,
    -4.51953125,
    -3.7734375,
    -4.66796875,
    -5.0,
    -4.10546875,
    -5.18359375,
    -4.2109375,
    -3.044921875,
    -5.2890625,
    -3.1015625,
    -3.35546875,
    -3.9453125,
    -4.66015625,
    -4.49609375,
    -6.0859375,
    -7.1796875,
    -6.796875,
    -7.78515625,
    -5.671875,
    -3.86328125,
    -7.00390625,
    -7.30078125,
    -2.421875,
    -7.2109375,
    -3.9296875,
    -7.765625,
    -5.015625,
    -2.861328125,
    -5.3046875,
    -3.6875,
    -5.828125,
    -4.1171875,
    -6.7421875,
    -3.630859375,
    -5.68359375,
    -4.28515625,
    -3.90234375,
    -2.71875,
    -4.1953125,
    -3.35546875,
    -4.37890625,
    -2.38671875,
    -3.80078125,
    -3.025390625,
    -4.4140625,
    -3.630859375,
    -4.125,
    -4.00390625,
    -3.470703125,
    -4.16796875,
    -4.12890625,
    -4.7578125,
    -3.369140625,
    -2.90625,
    -4.05078125,
    -5.3203125,
    -4.57421875,
    -3.033203125,
    -3.953125,
    -4.421875,
    -5.5703125,
    -3.53515625,
    -4.95703125,
    -4.06640625,
    -5.76953125,
    -4.1796875,
    -4.56640625,
    -4.796875,
    -5.89453125,
    -4.57421875,
    -3.69921875,
    -4.55078125,
    -5.4609375,
    -3.783203125,
    -3.732421875,
    -2.958984375,
    -4.19140625,
    -2.91015625,
    -3.61328125,
    -2.15234375,
    -5.0546875,
    -5.49609375,
    -2.40625,
    -2.69921875,
    -5.39453125,
    -5.6484375,
    -2.828125,
    -5.421875,
    -6.1796875,
    -4.95703125,
    -4.0859375,
    -3.580078125,
    -4.53515625,
    -5.8515625,
    -2.66796875,
    -3.501953125,
    -4.234375,
    -3.150390625,
    -2.4609375,
    -4.80859375,
    -5.01171875,
    -5.63671875,
    -5.40234375,
    -5.046875,
    -4.171875,
    -4.42578125,
    -2.67578125,
    -0.84375,
    -3.228515625,
    -2.546875,
    -1.5986328125,
    -1.5458984375,
    -1.689453125,
    -2.876953125,
    -0.55322265625,
    -1.7548828125,
    -2.076171875,
    -1.56640625,
    -0.99072265625,
    -2.734375,
    -1.875,
    -2.5859375,
    -2.28125,
    -3.56640625,
    -1.9599609375,
    -2.1328125,
    -2.458984375,
    -2.08203125,
    -2.951171875,
    -0.6171875,
    -1.7802734375,
    -3.369140625,
    -3.265625,
    -2.9140625,
    -2.236328125,
    -3.36328125,
    -2.302734375,
    -2.7578125,
    -2.185546875,
    -2.361328125,
    -1.908203125,
    -1.3037109375,
    -1.396484375,
    -2.42578125,
    -2.556640625,
    -0.5615234375,
    -1.693359375,
    -2.45703125,
    -1.4599609375,
    -1.5625,
    -1.5869140625,
    -2.171875,
    -1.4931640625,
    -2.07421875,
    -0.80078125,
    -1.037109375,
    -1.8564453125,
    -1.3046875,
    -1.7568359375,
    -2.236328125,
    -1.1298828125,
    -2.140625,
    -2.4609375,
    0.050506591796875,
    -1.94921875,
    -1.4033203125,
    -0.998046875,
    -0.010162353515625,
    -0.6826171875,
    -2.796875,
    -0.93115234375,
    -1.73046875,
    -1.890625,
    -1.052734375,
    -2.01953125,
    -1.306640625,
    -1.685546875,
    -0.818359375,
    -0.81640625,
    -0.156494140625,
    -0.0831298828125,
    -0.5751953125,
    0.97802734375,
    0.88525390625,
    -0.1826171875,
    -0.476806640625,
    -0.62548828125,
    0.34619140625,
    -0.830078125,
    0.16162109375,
    0.796875,
    1.275390625,
    -0.1676025390625,
    0.0517578125,
    -0.658203125,
    0.2327880859375,
    0.7607421875,
    -0.638671875,
    -0.1982421875,
    -0.2529296875,
    0.65673828125,
    0.53955078125,
    -0.537109375,
    -0.67919921875,
    2.20703125,
    -0.33740234375,
    -0.2802734375,
    -0.288818359375,
    0.0672607421875,
    -0.39599609375,
    -4.19140625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9854151010513306,
    0.010547679848968983,
    0.0031731673516333103,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0008640876621939242
   ],
   "p": [
    0.9862672686576843,
    0.010556801222264767,
    0.003175911493599415
   ],
   "p_oracle": [
    0.986133873462677,
    0.010701395571231842,
    0.0031647749710828066
   ],
   "abstain": 0.0008640876621939242,
   "abstain_oracle": 0.0008616404957138002,
   "abs_delta_abstain": 2.4471664801239967e-06,
   "oracle_margin": 0.9754324555397034,
   "max_abs_delta_p": 0.00014459434896707535,
   "mean_abs_delta_p": 9.63753554970026e-05,
   "max_abs_delta_raw_logits": 0.013790130615234375,
   "mean_abs_delta_raw_logits": 0.0015434705710504204,
   "logits_min": -7.78515625,
   "logits_max": 3.259765625,
   "raw_logits_min": -7.78515625,
   "raw_logits_max": 3.259765625,
   "raw_logits_absmax": 7.78515625,
   "oracle_logits_min": -7.7877583503723145,
   "oracle_logits_max": 3.259012222290039,
   "oracle_raw_logits_absmax": 7.7877583503723145,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5825922089861706,
   "seconds": 0.5825922089861706,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r04-drawer",
   "request_id": "r04",
   "question_id": "drawer",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 103,
   "nopts": 11,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.560546875,
    -1.6982421875,
    -2.029296875,
    -2.55078125,
    -2.37109375,
    -2.662109375,
    5.171875,
    -5.3046875,
    -2.615234375,
    -2.7265625,
    -3.37109375,
    -3.814453125,
    -4.8515625,
    -2.55859375,
    -3.98828125,
    -5.00390625,
    -4.8828125,
    -4.76171875,
    -2.486328125,
    -2.68359375,
    -3.78515625,
    -1.90625,
    -5.21484375,
    -6.2890625,
    -4.65234375,
    -5.05078125,
    -2.5625,
    -5.1640625,
    -4.4296875,
    -6.28515625,
    -4.86328125,
    -4.92578125,
    -5.7734375,
    -4.55859375,
    -2.955078125,
    -4.5546875,
    -5.53125,
    -6.71484375,
    -3.4375,
    -3.962890625,
    -6.328125,
    -4.33984375,
    -4.86328125,
    -5.04296875,
    -4.39453125,
    -4.5546875,
    -5.19140625,
    -5.1328125,
    -5.62109375,
    -3.44140625,
    -5.72265625,
    -4.359375,
    -1.66796875,
    -5.2734375,
    -3.1640625,
    -3.677734375,
    -4.44921875,
    -4.7421875,
    -4.76953125,
    -6.15234375,
    -7.453125,
    -6.40625,
    -7.58203125,
    -5.80859375,
    -3.80859375,
    -7.359375,
    -7.52734375,
    -3.046875,
    -7.8515625,
    -3.931640625,
    -7.546875,
    -4.93359375,
    -2.822265625,
    -6.17578125,
    -3.4140625,
    -5.9453125,
    -4.05078125,
    -6.86328125,
    -3.56640625,
    -5.4453125,
    -4.59765625,
    -4.00390625,
    -2.353515625,
    -4.2734375,
    -3.1484375,
    -4.55078125,
    -2.83984375,
    -4.16796875,
    -3.03125,
    -4.62109375,
    -3.53515625,
    -4.68359375,
    -4.5859375,
    -4.13671875,
    -5.046875,
    -4.859375,
    -5.1640625,
    -4.25,
    -3.880859375,
    -4.171875,
    -5.7109375,
    -4.60546875,
    -3.51953125,
    -4.13671875,
    -4.296875,
    -5.5625,
    -3.712890625,
    -5.40234375,
    -4.27734375,
    -5.42578125,
    -3.75,
    -4.48828125,
    -5.37109375,
    -6.609375,
    -5.08203125,
    -4.2265625,
    -4.49609375,
    -5.49609375,
    -4.0,
    -3.3984375,
    -2.927734375,
    -4.3046875,
    -3.294921875,
    -3.828125,
    -2.75390625,
    -5.6953125,
    -5.64453125,
    -2.67578125,
    -2.828125,
    -5.8359375,
    -5.9296875,
    -3.01953125,
    -5.7734375,
    -6.703125,
    -5.39453125,
    -4.07421875,
    -3.87109375,
    -4.01171875,
    -5.953125,
    -3.3671875,
    -3.708984375,
    -3.802734375,
    -4.046875,
    -3.060546875,
    -4.734375,
    -5.30078125,
    -5.8671875,
    -5.14453125,
    -5.29296875,
    -3.880859375,
    -5.44921875,
    -3.3671875,
    -0.92041015625,
    -3.740234375,
    -2.267578125,
    -1.64453125,
    -1.7353515625,
    -2.11328125,
    -3.48828125,
    -0.31298828125,
    -1.5869140625,
    -2.189453125,
    -1.6865234375,
    -0.96240234375,
    -3.869140625,
    -2.296875,
    -2.900390625,
    -2.7734375,
    -2.99609375,
    -1.5341796875,
    -2.486328125,
    -2.37890625,
    -1.97265625,
    -2.41015625,
    -0.9765625,
    -2.244140625,
    -3.548828125,
    -3.357421875,
    -3.212890625,
    -3.1875,
    -3.19921875,
    -2.96484375,
    -2.94140625,
    -2.609375,
    -3.373046875,
    -2.181640625,
    -2.076171875,
    -1.818359375,
    -1.9189453125,
    -2.509765625,
    -1.12109375,
    -1.5888671875,
    -2.82421875,
    -1.8564453125,
    -2.443359375,
    -1.9716796875,
    -2.470703125,
    -1.6708984375,
    -1.771484375,
    -1.552734375,
    -1.572265625,
    -2.20703125,
    -2.42578125,
    -1.9326171875,
    -2.189453125,
    -1.22265625,
    -2.2734375,
    -2.224609375,
    0.201904296875,
    -2.466796875,
    -0.78369140625,
    -0.703125,
    -1.0234375,
    -1.4150390625,
    -2.962890625,
    -0.951171875,
    -2.248046875,
    -2.646484375,
    -1.275390625,
    -1.615234375,
    -1.08203125,
    -1.634765625,
    -1.439453125,
    -1.041015625,
    -0.40234375,
    -0.01611328125,
    -0.890625,
    1.173828125,
    0.71630859375,
    -0.1278076171875,
    -0.42431640625,
    -0.466552734375,
    0.06292724609375,
    -0.736328125,
    0.40087890625,
    1.2802734375,
    1.4443359375,
    -0.6533203125,
    -0.310791015625,
    -0.85595703125,
    1.1181640625,
    0.39306640625,
    -0.953125,
    -0.1278076171875,
    -0.10906982421875,
    0.48291015625,
    0.353759765625,
    -0.91552734375,
    -0.83203125,
    1.1728515625,
    -0.0285797119140625,
    -0.04315185546875,
    0.11285400390625,
    0.001659393310546875,
    -0.50439453125,
    -3.490234375
   ],
   "raw256_argmax": 6,
   "option_argmax": 6,
   "oracle_argmax": 6,
   "argmax_agrees": true,
   "p_full": [
    0.0017148820916190743,
    0.0015057078562676907,
    0.0011013257317245007,
    0.0006729172309860587,
    0.0007974135805852711,
    0.0006057395949028432,
    0.9917620420455933,
    4.9898892029887065e-05,
    0.0006331662880256772,
    0.0005699570174328983,
    0.000310030416585505,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00027702777879312634
   ],
   "p": [
    0.0017153570661321282,
    0.0015061249723657966,
    0.001101630856283009,
    0.0006731036119163036,
    0.0007976344786584377,
    0.0006059074075892568,
    0.992036759853363,
    4.991271271137521e-05,
    0.0006333416677080095,
    0.0005701148766092956,
    0.00031011630198918283
   ],
   "p_oracle": [
    0.0017093284986913204,
    0.0014980518026277423,
    0.0010982141830027103,
    0.0006639480125159025,
    0.0007912743021734059,
    0.0005999383865855634,
    0.9920872449874878,
    4.973450268153101e-05,
    0.0006263956311158836,
    0.0005665535572916269,
    0.0003093348059337586
   ],
   "abstain": 0.00027702777879312634,
   "abstain_oracle": 0.0002797741908580065,
   "abs_delta_abstain": 2.7464120648801327e-06,
   "oracle_margin": 0.9903779029846191,
   "max_abs_delta_p": 5.048513412475586e-05,
   "mean_abs_delta_p": 9.177763951645995e-06,
   "max_abs_delta_raw_logits": 0.013637542724609375,
   "mean_abs_delta_raw_logits": 0.0024038960691541433,
   "logits_min": -7.8515625,
   "logits_max": 5.171875,
   "raw_logits_min": -7.8515625,
   "raw_logits_max": 5.171875,
   "raw_logits_absmax": 7.8515625,
   "oracle_logits_min": -7.854022979736328,
   "oracle_logits_max": 5.175121307373047,
   "oracle_raw_logits_absmax": 7.854022979736328,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5766323750140145,
   "seconds": 0.5766323750140145,
   "full_vocab_argmax_id": 6,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r04-key",
   "request_id": "r04",
   "question_id": "key",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 65,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -3.267578125,
    1.32421875,
    -0.73583984375,
    -1.23046875,
    -1.505859375,
    -2.693359375,
    -2.17578125,
    -3.158203125,
    -3.470703125,
    -1.8115234375,
    -2.25390625,
    -3.6953125,
    -4.1328125,
    -2.091796875,
    -2.994140625,
    -4.08203125,
    -4.05078125,
    -4.98046875,
    -2.892578125,
    -2.603515625,
    -3.498046875,
    -1.755859375,
    -5.03125,
    -6.37890625,
    -3.720703125,
    -4.484375,
    -2.62109375,
    -4.84765625,
    -3.646484375,
    -5.75,
    -4.0859375,
    -4.1015625,
    -5.55078125,
    -4.2421875,
    -2.4609375,
    -4.92578125,
    -5.0,
    -5.79296875,
    -4.125,
    -2.9375,
    -6.7109375,
    -4.05078125,
    -4.4765625,
    -5.76953125,
    -3.580078125,
    -4.4765625,
    -4.16015625,
    -4.20703125,
    -4.83203125,
    -4.16796875,
    -5.15625,
    -4.27734375,
    -2.494140625,
    -4.64453125,
    -2.353515625,
    -3.19140625,
    -3.4140625,
    -4.4140625,
    -4.140625,
    -5.140625,
    -6.859375,
    -5.90625,
    -7.1875,
    -5.953125,
    -3.302734375,
    -7.06640625,
    -7.08203125,
    -2.23828125,
    -7.51953125,
    -4.1171875,
    -7.40625,
    -4.8359375,
    -2.349609375,
    -5.7109375,
    -3.62109375,
    -5.640625,
    -4.140625,
    -6.31640625,
    -3.841796875,
    -5.265625,
    -4.09375,
    -3.775390625,
    -2.447265625,
    -3.712890625,
    -3.041015625,
    -3.890625,
    -2.345703125,
    -3.27734375,
    -2.72265625,
    -4.37890625,
    -3.45703125,
    -3.728515625,
    -3.279296875,
    -3.83203125,
    -4.3125,
    -4.75390625,
    -4.609375,
    -3.70703125,
    -3.486328125,
    -3.83984375,
    -5.203125,
    -4.55078125,
    -3.224609375,
    -3.564453125,
    -4.125,
    -5.33203125,
    -3.322265625,
    -4.9140625,
    -3.875,
    -5.03125,
    -4.1640625,
    -4.28515625,
    -4.52734375,
    -5.81640625,
    -4.28515625,
    -3.62109375,
    -4.42578125,
    -5.375,
    -3.892578125,
    -3.029296875,
    -3.072265625,
    -3.857421875,
    -3.13671875,
    -3.357421875,
    -2.416015625,
    -5.28515625,
    -4.8828125,
    -2.919921875,
    -2.78515625,
    -4.9375,
    -5.29296875,
    -2.982421875,
    -5.34765625,
    -5.74609375,
    -4.91015625,
    -4.2578125,
    -3.537109375,
    -4.3515625,
    -5.5,
    -2.615234375,
    -3.263671875,
    -3.8828125,
    -3.46875,
    -3.4296875,
    -4.40234375,
    -4.515625,
    -5.44140625,
    -4.8125,
    -4.78125,
    -3.9765625,
    -4.45703125,
    -2.88671875,
    -0.97412109375,
    -2.99609375,
    -2.123046875,
    -1.3515625,
    -1.6162109375,
    -1.765625,
    -2.98828125,
    -0.477294921875,
    -1.9873046875,
    -1.978515625,
    -1.3388671875,
    -1.333984375,
    -2.80078125,
    -2.171875,
    -2.494140625,
    -1.5673828125,
    -3.361328125,
    -1.9794921875,
    -1.6572265625,
    -2.29296875,
    -1.9853515625,
    -2.703125,
    -0.798828125,
    -1.71484375,
    -2.892578125,
    -3.173828125,
    -2.673828125,
    -3.18359375,
    -2.986328125,
    -2.74609375,
    -2.677734375,
    -1.927734375,
    -2.27734375,
    -1.4052734375,
    -1.6630859375,
    -1.6025390625,
    -1.6279296875,
    -2.09765625,
    -0.68798828125,
    -1.447265625,
    -2.40234375,
    -1.375,
    -1.5048828125,
    -1.9794921875,
    -1.900390625,
    -1.6484375,
    -1.431640625,
    -1.25,
    -1.24609375,
    -1.6220703125,
    -1.51953125,
    -1.6376953125,
    -2.20703125,
    -0.607421875,
    -2.4375,
    -2.328125,
    -0.132568359375,
    -1.875,
    -0.6328125,
    -1.05859375,
    -0.5380859375,
    -0.9560546875,
    -2.501953125,
    -1.0009765625,
    -1.5419921875,
    -2.205078125,
    -1.638671875,
    -1.5224609375,
    -1.4130859375,
    -1.5810546875,
    -1.470703125,
    -1.15625,
    -0.10784912109375,
    0.325927734375,
    -0.222900390625,
    0.93310546875,
    0.9921875,
    -0.1253662109375,
    -0.80224609375,
    -0.5546875,
    -0.13916015625,
    -0.8134765625,
    0.5234375,
    1.087890625,
    1.583984375,
    -0.748046875,
    -0.290771484375,
    -0.69189453125,
    0.75244140625,
    0.8447265625,
    -0.615234375,
    -0.12237548828125,
    -0.15771484375,
    0.4228515625,
    0.255126953125,
    -0.78076171875,
    -0.92626953125,
    1.453125,
    -0.07623291015625,
    -0.7919921875,
    -0.028411865234375,
    0.59521484375,
    -0.6708984375,
    -6.66015625
   ],
   "raw256_argmax": 236,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.010340511798858643,
    0.9893038272857666,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003556867013685405
   ],
   "p": [
    0.010344190523028374,
    0.9896557927131653
   ],
   "p_oracle": [
    0.010282500647008419,
    0.9897175431251526
   ],
   "abstain": 0.0003556867013685405,
   "abstain_oracle": 0.00035554287023842335,
   "abs_delta_abstain": 1.4383113011717796e-07,
   "oracle_margin": 0.9794350266456604,
   "max_abs_delta_p": 6.175041198730469e-05,
   "mean_abs_delta_p": 6.172014400362968e-05,
   "max_abs_delta_raw_logits": 0.007699489593505859,
   "mean_abs_delta_raw_logits": 0.001993904763367027,
   "logits_min": -7.51953125,
   "logits_max": 1.583984375,
   "raw_logits_min": -7.51953125,
   "raw_logits_max": 1.583984375,
   "raw_logits_absmax": 7.51953125,
   "oracle_logits_min": -7.522004127502441,
   "oracle_logits_max": 1.5854170322418213,
   "oracle_raw_logits_absmax": 7.522004127502441,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.36280595790594816,
   "seconds": 0.36280595790594816,
   "full_vocab_argmax_id": 236,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r05-task",
   "request_id": "r05",
   "question_id": "task",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 99,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    1.18359375,
    -1.4716796875,
    -4.47265625,
    -3.32421875,
    -3.16015625,
    -3.111328125,
    -2.943359375,
    -4.32421875,
    -5.00390625,
    -3.47265625,
    -2.908203125,
    -3.1015625,
    -5.19921875,
    -3.376953125,
    -3.912109375,
    -3.89453125,
    -4.2421875,
    -4.73046875,
    -3.396484375,
    -3.80859375,
    -3.5390625,
    -2.41796875,
    -5.375,
    -5.8515625,
    -4.20703125,
    -5.3984375,
    -2.912109375,
    -5.55859375,
    -4.46875,
    -6.36328125,
    -4.62109375,
    -4.34765625,
    -6.53515625,
    -4.3828125,
    -3.740234375,
    -5.33203125,
    -5.703125,
    -5.75,
    -4.31640625,
    -3.107421875,
    -6.1875,
    -5.22265625,
    -5.4765625,
    -5.5,
    -3.884765625,
    -5.0703125,
    -4.30078125,
    -4.94921875,
    -5.390625,
    -4.6171875,
    -5.5078125,
    -5.0,
    -3.7578125,
    -5.21875,
    -3.310546875,
    -3.703125,
    -4.0,
    -4.17578125,
    -4.66796875,
    -5.55859375,
    -7.578125,
    -7.0703125,
    -8.2265625,
    -6.27734375,
    -3.841796875,
    -7.5390625,
    -7.89453125,
    -2.837890625,
    -8.046875,
    -4.171875,
    -8.015625,
    -5.4296875,
    -2.703125,
    -5.5859375,
    -4.046875,
    -6.3125,
    -4.234375,
    -7.1171875,
    -3.96875,
    -6.03125,
    -4.265625,
    -4.15625,
    -2.90625,
    -4.7265625,
    -3.6796875,
    -4.57421875,
    -2.64453125,
    -4.03515625,
    -3.486328125,
    -4.921875,
    -3.916015625,
    -4.66015625,
    -4.46875,
    -3.83203125,
    -4.578125,
    -4.0234375,
    -5.02734375,
    -3.5390625,
    -2.97265625,
    -4.390625,
    -6.21875,
    -4.8359375,
    -3.515625,
    -4.69921875,
    -5.04296875,
    -6.19140625,
    -4.08984375,
    -5.5078125,
    -4.03515625,
    -5.93359375,
    -4.23046875,
    -5.0,
    -5.2421875,
    -6.2578125,
    -4.73046875,
    -3.57421875,
    -4.67578125,
    -6.08203125,
    -4.6796875,
    -3.96484375,
    -3.11328125,
    -4.12109375,
    -2.98828125,
    -3.91796875,
    -2.201171875,
    -5.38671875,
    -5.921875,
    -2.69921875,
    -3.0625,
    -5.91015625,
    -5.66796875,
    -3.716796875,
    -6.046875,
    -6.7109375,
    -5.18359375,
    -5.0234375,
    -3.73828125,
    -4.59375,
    -6.3828125,
    -2.89453125,
    -3.951171875,
    -4.3984375,
    -3.26953125,
    -3.056640625,
    -4.32421875,
    -5.24609375,
    -6.1796875,
    -5.8828125,
    -4.95703125,
    -4.30859375,
    -4.5,
    -2.837890625,
    -1.2744140625,
    -3.25390625,
    -2.08984375,
    -1.787109375,
    -1.3837890625,
    -1.6474609375,
    -2.958984375,
    -1.142578125,
    -2.1171875,
    -1.6982421875,
    -1.578125,
    -1.328125,
    -2.392578125,
    -2.2265625,
    -2.94140625,
    -2.49609375,
    -3.66015625,
    -2.37890625,
    -2.486328125,
    -2.658203125,
    -2.365234375,
    -3.16796875,
    -1.130859375,
    -1.9326171875,
    -3.90234375,
    -3.578125,
    -3.587890625,
    -3.06640625,
    -4.55078125,
    -3.076171875,
    -3.072265625,
    -3.07421875,
    -3.158203125,
    -2.490234375,
    -1.892578125,
    -2.17578125,
    -2.78515625,
    -2.65625,
    -1.5693359375,
    -2.220703125,
    -3.328125,
    -2.001953125,
    -1.8359375,
    -1.908203125,
    -2.552734375,
    -1.84375,
    -2.255859375,
    -1.3759765625,
    -1.748046875,
    -2.384765625,
    -1.2412109375,
    -2.326171875,
    -2.388671875,
    -1.8984375,
    -2.412109375,
    -2.552734375,
    -0.175537109375,
    -2.6171875,
    -1.76953125,
    -1.5498046875,
    -0.70068359375,
    -1.2275390625,
    -3.126953125,
    -1.4443359375,
    -2.412109375,
    -2.337890625,
    -1.7705078125,
    -2.501953125,
    -1.767578125,
    -1.2666015625,
    -1.2392578125,
    -0.654296875,
    -0.09893798828125,
    -0.2010498046875,
    -0.95703125,
    0.6142578125,
    0.448486328125,
    -0.2802734375,
    -0.264404296875,
    -0.53662109375,
    -0.01229095458984375,
    -0.85009765625,
    -0.2169189453125,
    1.2724609375,
    1.03125,
    -0.267333984375,
    -0.2890625,
    -0.70068359375,
    0.383544921875,
    0.6748046875,
    -0.479248046875,
    0.04693603515625,
    -0.154296875,
    0.48486328125,
    0.69775390625,
    -0.33935546875,
    0.00362396240234375,
    1.916015625,
    -0.658203125,
    -0.239501953125,
    -0.484619140625,
    -0.0312347412109375,
    -0.342041015625,
    -0.84375
   ],
   "raw256_argmax": 249,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.8107138872146606,
    0.06598784029483795,
    0.0038745864294469357,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.11942368000745773
   ],
   "p": [
    0.9206628203392029,
    0.07493710517883301,
    0.004400057718157768
   ],
   "p_oracle": [
    0.9156041741371155,
    0.07993196696043015,
    0.004463807214051485
   ],
   "abstain": 0.11942368000745773,
   "abstain_oracle": 0.13591159880161285,
   "abs_delta_abstain": 0.01648791879415512,
   "oracle_margin": 0.8356721997261047,
   "max_abs_delta_p": 0.005058646202087402,
   "mean_abs_delta_p": 0.003372419159859419,
   "max_abs_delta_raw_logits": 0.10010689496994019,
   "mean_abs_delta_raw_logits": 0.01547925331396982,
   "logits_min": -8.2265625,
   "logits_max": 1.916015625,
   "raw_logits_min": -8.2265625,
   "raw_logits_max": 1.916015625,
   "raw_logits_absmax": 8.2265625,
   "oracle_logits_min": -8.236438751220703,
   "oracle_logits_max": 1.9041804075241089,
   "oracle_raw_logits_absmax": 8.236438751220703,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5506762090371922,
   "seconds": 0.5506762090371922,
   "full_vocab_argmax_id": 249,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r05-leaves",
   "request_id": "r05",
   "question_id": "leaves",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 71,
   "nopts": 4,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.919921875,
    -1.736328125,
    3.671875,
    -2.46875,
    -2.64453125,
    -2.40625,
    -2.572265625,
    -3.1875,
    -2.740234375,
    -1.9716796875,
    -3.224609375,
    -3.015625,
    -4.16796875,
    -3.107421875,
    -3.5078125,
    -4.40234375,
    -3.9296875,
    -4.421875,
    -2.576171875,
    -3.13671875,
    -3.9296875,
    -1.55078125,
    -5.03515625,
    -5.87890625,
    -3.140625,
    -4.55078125,
    -3.2890625,
    -3.8046875,
    -3.91015625,
    -6.49609375,
    -4.1171875,
    -5.7734375,
    -4.59765625,
    -4.32421875,
    -2.87890625,
    -5.0,
    -5.28515625,
    -6.77734375,
    -4.87109375,
    -3.216796875,
    -5.8125,
    -4.11328125,
    -4.58984375,
    -6.0546875,
    -4.6875,
    -4.921875,
    -4.49609375,
    -4.171875,
    -5.19140625,
    -4.9296875,
    -5.2109375,
    -4.921875,
    -3.095703125,
    -4.83203125,
    -3.10546875,
    -3.66015625,
    -3.916015625,
    -5.015625,
    -5.140625,
    -6.390625,
    -7.4453125,
    -6.765625,
    -7.4453125,
    -6.04296875,
    -3.4453125,
    -7.078125,
    -7.3671875,
    -2.873046875,
    -7.86328125,
    -4.609375,
    -7.76953125,
    -5.3359375,
    -2.904296875,
    -5.56640625,
    -2.87890625,
    -6.11328125,
    -3.84375,
    -6.59375,
    -4.31640625,
    -5.671875,
    -4.67578125,
    -3.736328125,
    -2.630859375,
    -4.40625,
    -3.271484375,
    -4.46875,
    -2.544921875,
    -4.171875,
    -2.986328125,
    -4.453125,
    -3.658203125,
    -3.83203125,
    -4.32421875,
    -3.328125,
    -5.0,
    -4.6796875,
    -5.1171875,
    -3.369140625,
    -3.208984375,
    -4.49609375,
    -5.6484375,
    -4.40625,
    -3.91015625,
    -3.638671875,
    -4.42578125,
    -5.62890625,
    -3.48046875,
    -5.3125,
    -4.328125,
    -5.76171875,
    -3.9921875,
    -4.53515625,
    -5.3671875,
    -5.81640625,
    -4.3828125,
    -3.748046875,
    -4.44921875,
    -5.8046875,
    -3.859375,
    -3.302734375,
    -3.60546875,
    -4.25,
    -3.00390625,
    -3.060546875,
    -2.53515625,
    -5.14453125,
    -5.43359375,
    -3.04296875,
    -2.517578125,
    -5.25,
    -5.26953125,
    -2.55078125,
    -5.43359375,
    -6.08984375,
    -5.2734375,
    -4.71875,
    -3.546875,
    -4.4375,
    -5.70703125,
    -3.396484375,
    -3.03125,
    -4.015625,
    -3.494140625,
    -2.8515625,
    -4.140625,
    -4.50390625,
    -5.421875,
    -5.01171875,
    -4.91796875,
    -4.38671875,
    -4.85546875,
    -2.83984375,
    -0.81201171875,
    -3.484375,
    -2.06640625,
    -0.98974609375,
    -1.6064453125,
    -1.7080078125,
    -2.873046875,
    0.054779052734375,
    -1.7548828125,
    -1.83203125,
    -1.4765625,
    -1.544921875,
    -3.068359375,
    -2.158203125,
    -3.251953125,
    -1.923828125,
    -3.55078125,
    -1.8818359375,
    -2.072265625,
    -2.232421875,
    -1.9541015625,
    -2.64453125,
    -0.62060546875,
    -1.859375,
    -2.83203125,
    -3.28125,
    -2.73046875,
    -2.640625,
    -2.75390625,
    -2.79296875,
    -2.955078125,
    -1.962890625,
    -2.728515625,
    -1.74609375,
    -2.376953125,
    -1.283203125,
    -1.7275390625,
    -2.966796875,
    -0.7060546875,
    -1.806640625,
    -2.6484375,
    -1.609375,
    -1.8984375,
    -2.2734375,
    -1.701171875,
    -1.7783203125,
    -1.701171875,
    -1.2744140625,
    -1.1435546875,
    -2.060546875,
    -1.970703125,
    -1.890625,
    -2.712890625,
    -0.88330078125,
    -2.537109375,
    -2.30859375,
    -0.07415771484375,
    -2.025390625,
    -0.99169921875,
    -0.994140625,
    -0.2232666015625,
    -1.4111328125,
    -3.03125,
    -0.86572265625,
    -1.53515625,
    -2.177734375,
    -1.1513671875,
    -1.55859375,
    -0.62841796875,
    -1.984375,
    -1.0771484375,
    -1.4560546875,
    -0.1142578125,
    -0.03369140625,
    -0.2724609375,
    1.052734375,
    1.013671875,
    -0.1995849609375,
    -0.27099609375,
    -0.5595703125,
    0.01117706298828125,
    -0.8251953125,
    0.677734375,
    0.734375,
    1.25,
    -0.473388671875,
    0.0302886962890625,
    -1.0400390625,
    0.60546875,
    0.3642578125,
    -0.55029296875,
    -0.024627685546875,
    -0.028564453125,
    1.169921875,
    0.54638671875,
    -0.7744140625,
    -0.81103515625,
    1.96875,
    -0.400390625,
    -0.50146484375,
    0.050933837890625,
    0.457763671875,
    0.01690673828125,
    -5.6015625
   ],
   "raw256_argmax": 2,
   "option_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "p_full": [
    0.00500765535980463,
    0.005956063512712717,
    0.9858999848365784,
    0.002981684636324644,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0001545729610370472
   ],
   "p": [
    0.005008429754525423,
    0.005956984590739012,
    0.9860524535179138,
    0.0029821456409990788
   ],
   "p_oracle": [
    0.005044578108936548,
    0.00600813515484333,
    0.9859573841094971,
    0.0029899454675614834
   ],
   "abstain": 0.0001545729610370472,
   "abstain_oracle": 0.0001540259108878672,
   "abs_delta_abstain": 5.470501491799951e-07,
   "oracle_margin": 0.9799492359161377,
   "max_abs_delta_p": 9.506940841674805e-05,
   "mean_abs_delta_p": 4.754203837364912e-05,
   "max_abs_delta_raw_logits": 0.006491661071777344,
   "mean_abs_delta_raw_logits": 0.0013574733748100698,
   "logits_min": -7.86328125,
   "logits_max": 3.671875,
   "raw_logits_min": -7.86328125,
   "raw_logits_max": 3.671875,
   "raw_logits_absmax": 7.86328125,
   "oracle_logits_min": -7.863582611083984,
   "oracle_logits_max": 3.669034242630005,
   "oracle_raw_logits_absmax": 7.863582611083984,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.39493245794437826,
   "seconds": 0.39493245794437826,
   "full_vocab_argmax_id": 2,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r05-dry",
   "request_id": "r05",
   "question_id": "dry",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 73,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    0.06463623046875,
    0.177978515625,
    -1.271484375,
    -2.2265625,
    -2.1171875,
    -2.931640625,
    -2.623046875,
    -3.029296875,
    -4.58203125,
    -2.64453125,
    -1.9736328125,
    -3.33203125,
    -3.802734375,
    -2.4296875,
    -3.12109375,
    -3.95703125,
    -4.01171875,
    -5.3359375,
    -3.3125,
    -3.138671875,
    -3.515625,
    -2.05078125,
    -5.09375,
    -6.078125,
    -3.43359375,
    -4.7578125,
    -2.728515625,
    -4.7578125,
    -3.65625,
    -5.4453125,
    -4.01953125,
    -3.962890625,
    -5.4921875,
    -4.03515625,
    -2.64453125,
    -4.68359375,
    -5.2109375,
    -5.4609375,
    -4.578125,
    -2.587890625,
    -6.34375,
    -4.1953125,
    -4.6015625,
    -5.51953125,
    -3.25390625,
    -4.50390625,
    -3.8203125,
    -4.53515625,
    -4.59765625,
    -4.1875,
    -5.10546875,
    -4.4453125,
    -3.033203125,
    -4.75390625,
    -2.765625,
    -3.439453125,
    -3.62890625,
    -4.3984375,
    -4.07421875,
    -5.515625,
    -7.203125,
    -6.38671875,
    -7.8203125,
    -5.83203125,
    -3.2890625,
    -7.17578125,
    -7.19140625,
    -2.291015625,
    -7.484375,
    -4.23828125,
    -7.52734375,
    -4.8359375,
    -2.537109375,
    -5.3671875,
    -3.724609375,
    -5.58984375,
    -4.26953125,
    -6.27734375,
    -3.59375,
    -5.43359375,
    -4.140625,
    -3.49609375,
    -2.38671875,
    -4.0625,
    -3.2890625,
    -3.953125,
    -2.458984375,
    -3.39453125,
    -3.076171875,
    -4.16796875,
    -3.150390625,
    -3.7421875,
    -3.390625,
    -3.755859375,
    -4.3515625,
    -4.27734375,
    -4.55859375,
    -3.357421875,
    -2.640625,
    -3.859375,
    -5.1640625,
    -4.87109375,
    -3.07421875,
    -3.896484375,
    -4.0625,
    -5.48828125,
    -3.103515625,
    -4.71484375,
    -3.84375,
    -5.3203125,
    -4.140625,
    -4.53515625,
    -4.3828125,
    -5.60546875,
    -4.2265625,
    -3.17578125,
    -4.53125,
    -5.125,
    -3.87109375,
    -3.31640625,
    -3.193359375,
    -4.07421875,
    -2.77734375,
    -3.375,
    -2.359375,
    -4.83203125,
    -5.04296875,
    -2.6328125,
    -2.576171875,
    -4.71484375,
    -5.17578125,
    -2.59375,
    -5.0859375,
    -5.8671875,
    -5.18359375,
    -4.37109375,
    -3.396484375,
    -4.1796875,
    -5.46484375,
    -2.30078125,
    -3.337890625,
    -3.953125,
    -3.00390625,
    -2.955078125,
    -4.3359375,
    -4.72265625,
    -5.53515625,
    -4.96875,
    -4.80078125,
    -3.83984375,
    -4.1328125,
    -2.30859375,
    -0.80810546875,
    -2.5859375,
    -1.966796875,
    -0.94873046875,
    -1.3828125,
    -1.5595703125,
    -2.734375,
    -0.7568359375,
    -1.76171875,
    -1.654296875,
    -0.857421875,
    -1.1416015625,
    -2.533203125,
    -1.9482421875,
    -2.361328125,
    -1.66015625,
    -3.474609375,
    -1.8642578125,
    -1.736328125,
    -2.099609375,
    -1.74609375,
    -2.58984375,
    -0.71484375,
    -1.6201171875,
    -3.185546875,
    -3.01953125,
    -2.5703125,
    -2.603515625,
    -3.09765625,
    -2.158203125,
    -2.3515625,
    -1.9130859375,
    -2.07421875,
    -1.673828125,
    -1.4013671875,
    -1.390625,
    -2.005859375,
    -1.9638671875,
    -0.4873046875,
    -1.771484375,
    -2.326171875,
    -1.2099609375,
    -1.2587890625,
    -1.587890625,
    -1.8759765625,
    -1.37109375,
    -1.6435546875,
    -0.9609375,
    -1.044921875,
    -1.63671875,
    -1.2021484375,
    -1.296875,
    -1.9794921875,
    -0.86083984375,
    -2.251953125,
    -2.4609375,
    -0.129638671875,
    -1.693359375,
    -0.66748046875,
    -1.02734375,
    -0.099365234375,
    -0.8525390625,
    -2.478515625,
    -0.7978515625,
    -1.2509765625,
    -2.123046875,
    -1.3408203125,
    -1.6376953125,
    -1.4990234375,
    -1.630859375,
    -1.359375,
    -0.806640625,
    -0.0166015625,
    0.35205078125,
    -0.4189453125,
    0.9013671875,
    0.91845703125,
    -0.263671875,
    -0.304443359375,
    -0.346923828125,
    -0.0770263671875,
    -0.94091796875,
    0.2607421875,
    1.017578125,
    1.3935546875,
    -0.339111328125,
    -0.226318359375,
    -0.65087890625,
    0.6162109375,
    0.56298828125,
    -0.45654296875,
    0.1588134765625,
    0.08050537109375,
    0.43701171875,
    0.57080078125,
    -0.54443359375,
    -0.62890625,
    1.9365234375,
    -0.3623046875,
    -0.5751953125,
    -0.341796875,
    0.352294921875,
    -0.701171875,
    -7.91796875
   ],
   "raw256_argmax": 249,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.4718044102191925,
    0.5280256867408752,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00016992742894217372
   ],
   "p": [
    0.4718845784664154,
    0.5281153917312622
   ],
   "p_oracle": [
    0.4691595137119293,
    0.5308405160903931
   ],
   "abstain": 0.00016992742894217372,
   "abstain_oracle": 0.00016973857418633997,
   "abs_delta_abstain": 1.88854755833745e-07,
   "oracle_margin": 0.061681002378463745,
   "max_abs_delta_p": 0.0027251243591308594,
   "mean_abs_delta_p": 0.0027250945568084717,
   "max_abs_delta_raw_logits": 0.009974956512451172,
   "mean_abs_delta_raw_logits": 0.0014678033767268062,
   "logits_min": -7.91796875,
   "logits_max": 1.9365234375,
   "raw_logits_min": -7.91796875,
   "raw_logits_max": 1.9365234375,
   "raw_logits_absmax": 7.91796875,
   "oracle_logits_min": -7.923232078552246,
   "oracle_logits_max": 1.9354100227355957,
   "oracle_raw_logits_absmax": 7.923232078552246,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.40622041607275605,
   "seconds": 0.40622041607275605,
   "full_vocab_argmax_id": 249,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r06-phase",
   "request_id": "r06",
   "question_id": "phase",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 96,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.794921875,
    3.2109375,
    -2.443359375,
    -2.740234375,
    -2.01171875,
    -2.46484375,
    -2.373046875,
    -3.560546875,
    -3.748046875,
    -2.43359375,
    -2.93359375,
    -3.845703125,
    -4.94921875,
    -3.029296875,
    -4.1484375,
    -4.27734375,
    -3.744140625,
    -4.234375,
    -2.90234375,
    -2.796875,
    -3.1484375,
    -1.818359375,
    -5.03125,
    -6.7421875,
    -3.703125,
    -4.96875,
    -2.34375,
    -5.1015625,
    -4.3984375,
    -6.484375,
    -4.44921875,
    -4.5625,
    -6.03125,
    -4.80859375,
    -3.06640625,
    -5.21484375,
    -5.19140625,
    -5.83984375,
    -4.11328125,
    -2.9140625,
    -6.828125,
    -4.25,
    -4.75,
    -5.80078125,
    -4.14453125,
    -4.81640625,
    -4.375,
    -4.296875,
    -5.15234375,
    -4.390625,
    -5.5859375,
    -4.83203125,
    -2.509765625,
    -4.87890625,
    -2.779296875,
    -3.33984375,
    -3.74609375,
    -4.5859375,
    -4.77734375,
    -5.73046875,
    -7.3515625,
    -6.23828125,
    -7.640625,
    -6.44140625,
    -3.3125,
    -7.26953125,
    -7.3671875,
    -2.408203125,
    -7.48046875,
    -3.828125,
    -7.734375,
    -5.08203125,
    -2.509765625,
    -5.84375,
    -3.736328125,
    -5.875,
    -3.93359375,
    -6.52734375,
    -4.14453125,
    -5.55078125,
    -4.296875,
    -4.01953125,
    -2.6015625,
    -4.1640625,
    -3.38671875,
    -3.802734375,
    -2.46484375,
    -3.771484375,
    -2.978515625,
    -4.56640625,
    -3.259765625,
    -3.958984375,
    -4.234375,
    -3.908203125,
    -4.390625,
    -4.546875,
    -5.03515625,
    -3.751953125,
    -3.798828125,
    -4.140625,
    -5.81640625,
    -4.3671875,
    -3.39453125,
    -4.00390625,
    -4.17578125,
    -5.97265625,
    -4.15625,
    -5.46875,
    -3.9375,
    -5.453125,
    -4.2265625,
    -4.5234375,
    -4.9765625,
    -6.35546875,
    -4.6484375,
    -3.912109375,
    -4.4765625,
    -5.7265625,
    -4.1953125,
    -3.498046875,
    -3.34375,
    -3.955078125,
    -3.189453125,
    -3.765625,
    -2.3984375,
    -5.61328125,
    -5.59375,
    -2.994140625,
    -3.087890625,
    -5.2890625,
    -5.48828125,
    -3.34765625,
    -5.37890625,
    -5.8046875,
    -4.76953125,
    -4.375,
    -3.861328125,
    -4.88671875,
    -5.66796875,
    -2.970703125,
    -3.521484375,
    -4.06640625,
    -3.58203125,
    -2.939453125,
    -4.75,
    -5.26953125,
    -5.84375,
    -5.0,
    -4.60546875,
    -4.4140625,
    -4.72265625,
    -3.0859375,
    -0.95166015625,
    -3.353515625,
    -2.55859375,
    -1.5712890625,
    -1.8798828125,
    -2.0390625,
    -3.396484375,
    -0.309326171875,
    -2.037109375,
    -2.001953125,
    -1.6728515625,
    -1.3544921875,
    -2.46875,
    -2.21484375,
    -2.7421875,
    -1.8330078125,
    -3.33203125,
    -1.8583984375,
    -2.1875,
    -2.474609375,
    -2.390625,
    -2.966796875,
    -0.703125,
    -1.75390625,
    -3.25390625,
    -3.205078125,
    -2.787109375,
    -3.13671875,
    -3.345703125,
    -2.912109375,
    -2.447265625,
    -2.04296875,
    -2.419921875,
    -1.3935546875,
    -1.0234375,
    -1.5869140625,
    -1.7919921875,
    -1.90625,
    -0.74462890625,
    -1.62109375,
    -2.708984375,
    -1.6240234375,
    -1.470703125,
    -2.0234375,
    -1.9267578125,
    -1.76953125,
    -1.4716796875,
    -1.1123046875,
    -1.01171875,
    -1.7607421875,
    -1.712890625,
    -1.798828125,
    -2.30859375,
    -0.88818359375,
    -2.546875,
    -2.478515625,
    -0.456787109375,
    -2.205078125,
    -0.82958984375,
    -1.193359375,
    -0.72998046875,
    -1.3388671875,
    -2.4453125,
    -1.21875,
    -1.6318359375,
    -2.15625,
    -1.6796875,
    -1.7451171875,
    -1.3681640625,
    -1.6806640625,
    -1.359375,
    -1.18359375,
    -0.2008056640625,
    -0.03167724609375,
    -0.4609375,
    0.9931640625,
    0.96533203125,
    -0.012237548828125,
    -0.7998046875,
    -0.47607421875,
    0.21533203125,
    -0.681640625,
    0.5048828125,
    1.2109375,
    1.4580078125,
    -0.59814453125,
    0.09454345703125,
    -0.888671875,
    0.381103515625,
    0.7060546875,
    -0.6923828125,
    -0.136962890625,
    -0.03729248046875,
    0.56689453125,
    0.388427734375,
    -0.357421875,
    -0.99365234375,
    1.5712890625,
    -0.06427001953125,
    -0.85009765625,
    -0.1697998046875,
    0.377197265625,
    -0.5048828125,
    -5.046875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.008712601847946644,
    0.9861620664596558,
    0.0047217984683811665,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0004035886377096176
   ],
   "p": [
    0.008716119453310966,
    0.9865601658821106,
    0.004723704420030117
   ],
   "p_oracle": [
    0.008610899560153484,
    0.9867118000984192,
    0.0046772826462984085
   ],
   "abstain": 0.0004035886377096176,
   "abstain_oracle": 0.0003977194719482213,
   "abs_delta_abstain": 5.869165761396289e-06,
   "oracle_margin": 0.9781008958816528,
   "max_abs_delta_p": 0.00015163421630859375,
   "mean_abs_delta_p": 0.00010109196106592815,
   "max_abs_delta_raw_logits": 0.0113067626953125,
   "mean_abs_delta_raw_logits": 0.002222742623416707,
   "logits_min": -7.734375,
   "logits_max": 3.2109375,
   "raw_logits_min": -7.734375,
   "raw_logits_max": 3.2109375,
   "raw_logits_absmax": 7.734375,
   "oracle_logits_min": -7.740938186645508,
   "oracle_logits_max": 3.2222442626953125,
   "oracle_raw_logits_absmax": 7.740938186645508,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.539219000027515,
   "seconds": 0.539219000027515,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r06-priority",
   "request_id": "r06",
   "question_id": "priority",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 68,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -3.203125,
    -2.748046875,
    3.15625,
    -1.51171875,
    -2.6171875,
    -2.744140625,
    -2.392578125,
    -2.59765625,
    -2.7734375,
    -1.708984375,
    -2.892578125,
    -2.474609375,
    -4.46484375,
    -2.578125,
    -3.37890625,
    -4.14453125,
    -3.7109375,
    -4.4375,
    -2.51953125,
    -2.4375,
    -4.0859375,
    -1.4306640625,
    -5.2265625,
    -5.4765625,
    -3.3984375,
    -4.578125,
    -3.396484375,
    -4.08984375,
    -3.802734375,
    -5.74609375,
    -4.4453125,
    -5.0,
    -4.875,
    -4.0703125,
    -2.75,
    -4.91015625,
    -5.35546875,
    -6.54296875,
    -4.74609375,
    -3.21875,
    -5.96875,
    -4.16015625,
    -4.84765625,
    -6.24609375,
    -4.3125,
    -4.89453125,
    -4.41796875,
    -4.32421875,
    -4.88671875,
    -4.75,
    -5.203125,
    -4.76953125,
    -2.77734375,
    -4.84765625,
    -2.935546875,
    -3.638671875,
    -4.09375,
    -5.2265625,
    -5.15234375,
    -6.0703125,
    -7.4140625,
    -6.63671875,
    -7.43359375,
    -5.91015625,
    -3.306640625,
    -7.0703125,
    -7.34765625,
    -2.958984375,
    -7.57421875,
    -4.58984375,
    -7.9453125,
    -5.23046875,
    -2.701171875,
    -5.5234375,
    -2.78515625,
    -6.26953125,
    -3.833984375,
    -6.484375,
    -4.2421875,
    -5.546875,
    -4.45703125,
    -3.8125,
    -2.455078125,
    -4.5390625,
    -3.3125,
    -4.58203125,
    -2.5859375,
    -4.12109375,
    -3.025390625,
    -4.3046875,
    -3.42578125,
    -3.7578125,
    -4.234375,
    -3.24609375,
    -4.89453125,
    -4.609375,
    -4.953125,
    -3.373046875,
    -2.998046875,
    -4.57421875,
    -5.6328125,
    -4.515625,
    -3.724609375,
    -3.724609375,
    -4.33984375,
    -5.46484375,
    -3.31640625,
    -5.14453125,
    -4.06640625,
    -5.53515625,
    -3.986328125,
    -4.42578125,
    -4.94921875,
    -5.5703125,
    -4.46484375,
    -3.6796875,
    -4.30078125,
    -5.7890625,
    -3.919921875,
    -3.12109375,
    -3.337890625,
    -4.23046875,
    -2.736328125,
    -3.072265625,
    -2.466796875,
    -5.09375,
    -5.13671875,
    -2.90625,
    -2.583984375,
    -5.2578125,
    -5.359375,
    -2.41796875,
    -5.53515625,
    -6.15234375,
    -5.5234375,
    -4.66796875,
    -3.64453125,
    -4.37109375,
    -5.70703125,
    -3.017578125,
    -2.95703125,
    -4.04296875,
    -3.53125,
    -2.77734375,
    -4.03515625,
    -4.44921875,
    -5.2734375,
    -5.12109375,
    -5.0390625,
    -4.4140625,
    -4.62890625,
    -2.78515625,
    -0.5810546875,
    -3.5078125,
    -2.1953125,
    -1.0478515625,
    -1.44921875,
    -1.6904296875,
    -2.783203125,
    -0.11053466796875,
    -1.654296875,
    -1.9814453125,
    -1.2509765625,
    -1.3583984375,
    -3.134765625,
    -2.26953125,
    -3.001953125,
    -1.8544921875,
    -3.650390625,
    -1.90625,
    -2.01953125,
    -2.119140625,
    -1.9267578125,
    -2.59375,
    -0.59228515625,
    -1.62109375,
    -2.798828125,
    -3.1875,
    -2.72265625,
    -2.794921875,
    -2.740234375,
    -2.419921875,
    -2.986328125,
    -2.064453125,
    -2.876953125,
    -1.6279296875,
    -2.181640625,
    -1.3935546875,
    -1.4833984375,
    -2.576171875,
    -0.6298828125,
    -1.619140625,
    -2.52734375,
    -1.4384765625,
    -1.8251953125,
    -2.3046875,
    -1.5537109375,
    -1.5908203125,
    -1.619140625,
    -1.224609375,
    -0.99658203125,
    -2.0546875,
    -1.6201171875,
    -1.7548828125,
    -2.61328125,
    -0.75927734375,
    -2.2265625,
    -2.44921875,
    0.0208740234375,
    -1.7880859375,
    -0.92578125,
    -1.0341796875,
    -0.04876708984375,
    -1.423828125,
    -3.03125,
    -0.85205078125,
    -1.4111328125,
    -2.033203125,
    -1.294921875,
    -1.5498046875,
    -0.802734375,
    -1.80859375,
    -1.2783203125,
    -1.435546875,
    0.08380126953125,
    -0.07220458984375,
    -0.403076171875,
    1.05859375,
    1.19921875,
    -0.184326171875,
    -0.35498046875,
    -0.6083984375,
    -0.146484375,
    -1.056640625,
    0.75634765625,
    0.845703125,
    1.2158203125,
    -0.416015625,
    -0.208984375,
    -0.95068359375,
    0.62255859375,
    0.4609375,
    -0.57958984375,
    0.1640625,
    -0.08819580078125,
    1.0673828125,
    0.4443359375,
    -0.70947265625,
    -0.572265625,
    1.75,
    -0.299560546875,
    -0.5595703125,
    -0.297607421875,
    0.38671875,
    0.047637939453125,
    -6.08984375
   ],
   "raw256_argmax": 2,
   "option_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "p_full": [
    0.0024440435227006674,
    0.003756800899282098,
    0.9936392903327942,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00015986309153959155
   ],
   "p": [
    0.0024444342125207186,
    0.003757401602342725,
    0.9937981367111206
   ],
   "p_oracle": [
    0.002428818494081497,
    0.003745054127648473,
    0.993826150894165
   ],
   "abstain": 0.00015986309153959155,
   "abstain_oracle": 0.000158899201778695,
   "abs_delta_abstain": 9.638897608965635e-07,
   "oracle_margin": 0.9900810718536377,
   "max_abs_delta_p": 2.8014183044433594e-05,
   "mean_abs_delta_p": 1.8659125392635662e-05,
   "max_abs_delta_raw_logits": 0.007355690002441406,
   "mean_abs_delta_raw_logits": 0.0013954273017589003,
   "logits_min": -7.9453125,
   "logits_max": 3.15625,
   "raw_logits_min": -7.9453125,
   "raw_logits_max": 3.15625,
   "raw_logits_absmax": 7.9453125,
   "oracle_logits_min": -7.948171615600586,
   "oracle_logits_max": 3.1572537422180176,
   "oracle_raw_logits_absmax": 7.948171615600586,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3810751249548048,
   "seconds": 0.3810751249548048,
   "full_vocab_argmax_id": 2,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r06-approved",
   "request_id": "r06",
   "question_id": "approved",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 90,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    3.357421875,
    -2.35546875,
    -2.203125,
    -2.779296875,
    -2.361328125,
    -2.26953125,
    -2.552734375,
    -3.83203125,
    -3.13671875,
    -2.4296875,
    -2.75,
    -3.4296875,
    -4.51171875,
    -2.61328125,
    -3.953125,
    -4.11328125,
    -4.03515625,
    -4.58984375,
    -2.69921875,
    -3.208984375,
    -3.24609375,
    -1.4462890625,
    -4.984375,
    -5.98046875,
    -3.560546875,
    -4.19140625,
    -2.796875,
    -4.96875,
    -4.02734375,
    -6.47265625,
    -4.05078125,
    -4.8828125,
    -6.09375,
    -4.296875,
    -2.986328125,
    -4.77734375,
    -5.015625,
    -6.33203125,
    -4.203125,
    -3.556640625,
    -6.35546875,
    -4.40625,
    -5.2421875,
    -6.17578125,
    -4.015625,
    -4.8046875,
    -4.1796875,
    -4.88671875,
    -5.27734375,
    -4.11328125,
    -5.37890625,
    -4.359375,
    -3.25390625,
    -5.484375,
    -3.12109375,
    -3.447265625,
    -4.09375,
    -4.7578125,
    -4.73828125,
    -6.6171875,
    -7.3359375,
    -6.8046875,
    -7.92578125,
    -5.859375,
    -3.8671875,
    -7.20703125,
    -7.61328125,
    -2.412109375,
    -7.3515625,
    -4.09375,
    -8.078125,
    -5.3046875,
    -3.255859375,
    -5.48828125,
    -3.69921875,
    -6.30859375,
    -4.08984375,
    -7.0390625,
    -3.8984375,
    -5.9375,
    -4.8203125,
    -4.16015625,
    -2.953125,
    -4.5390625,
    -3.419921875,
    -4.3984375,
    -2.384765625,
    -4.05859375,
    -3.19921875,
    -4.61328125,
    -3.603515625,
    -4.25,
    -4.3125,
    -3.4453125,
    -4.50390625,
    -4.41796875,
    -4.953125,
    -3.44921875,
    -3.07421875,
    -4.13671875,
    -5.59375,
    -4.74609375,
    -3.27734375,
    -4.0625,
    -4.58984375,
    -5.66796875,
    -3.87109375,
    -5.2578125,
    -4.41015625,
    -5.9765625,
    -4.62890625,
    -4.71875,
    -5.1015625,
    -6.0,
    -4.8515625,
    -3.87109375,
    -4.57421875,
    -5.80859375,
    -3.82421875,
    -3.8125,
    -3.212890625,
    -4.33984375,
    -3.060546875,
    -3.927734375,
    -2.2734375,
    -5.27734375,
    -5.76171875,
    -2.716796875,
    -3.048828125,
    -5.58203125,
    -6.1484375,
    -2.78125,
    -5.796875,
    -6.41015625,
    -4.8515625,
    -3.990234375,
    -3.66796875,
    -4.83203125,
    -6.0234375,
    -2.86328125,
    -3.515625,
    -4.2421875,
    -3.4140625,
    -2.408203125,
    -4.9609375,
    -5.13671875,
    -5.796875,
    -5.47265625,
    -5.2578125,
    -4.52734375,
    -4.5234375,
    -2.7421875,
    -1.0341796875,
    -3.68359375,
    -2.923828125,
    -1.5966796875,
    -1.78125,
    -1.8037109375,
    -2.8359375,
    -0.4208984375,
    -1.8505859375,
    -2.16796875,
    -1.6572265625,
    -0.96337890625,
    -2.869140625,
    -1.794921875,
    -2.748046875,
    -2.45703125,
    -3.775390625,
    -1.8447265625,
    -2.197265625,
    -2.638671875,
    -2.35546875,
    -3.02734375,
    -0.62255859375,
    -1.978515625,
    -3.857421875,
    -3.26953125,
    -2.921875,
    -2.173828125,
    -3.21875,
    -2.4921875,
    -2.892578125,
    -2.115234375,
    -2.552734375,
    -1.7509765625,
    -1.353515625,
    -1.55078125,
    -2.123046875,
    -2.693359375,
    -0.58154296875,
    -1.6728515625,
    -2.552734375,
    -1.6005859375,
    -1.83203125,
    -1.7509765625,
    -2.052734375,
    -1.7998046875,
    -2.01953125,
    -1.0107421875,
    -1.2138671875,
    -1.9072265625,
    -1.48046875,
    -1.931640625,
    -2.34375,
    -1.03515625,
    -2.140625,
    -2.388671875,
    0.15673828125,
    -1.943359375,
    -1.3408203125,
    -0.99951171875,
    0.00655364990234375,
    -0.6181640625,
    -2.744140625,
    -0.99267578125,
    -1.69921875,
    -1.8779296875,
    -1.16015625,
    -1.78125,
    -0.892578125,
    -1.60546875,
    -0.74658203125,
    -1.1181640625,
    -0.251220703125,
    -0.245361328125,
    -0.556640625,
    1.1005859375,
    0.8134765625,
    -0.09344482421875,
    -0.486083984375,
    -0.576171875,
    0.6337890625,
    -0.740234375,
    0.240966796875,
    0.45068359375,
    1.2158203125,
    -0.3232421875,
    0.211669921875,
    -0.83447265625,
    0.39501953125,
    0.7841796875,
    -0.939453125,
    -0.517578125,
    -0.1776123046875,
    1.015625,
    0.60546875,
    -0.72900390625,
    -0.57080078125,
    2.30859375,
    -0.29638671875,
    -0.10516357421875,
    -0.462646484375,
    0.1961669921875,
    -0.34814453125,
    -4.515625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9961806535720825,
    0.003419307991862297,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00040003712638281286
   ],
   "p": [
    0.9965793490409851,
    0.0034206765703856945
   ],
   "p_oracle": [
    0.996588408946991,
    0.003411603160202503
   ],
   "abstain": 0.00040003712638281286,
   "abstain_oracle": 0.00040132785215973854,
   "abs_delta_abstain": 1.290725776925683e-06,
   "oracle_margin": 0.9931768178939819,
   "max_abs_delta_p": 9.0734101831913e-06,
   "mean_abs_delta_p": 9.066658094525337e-06,
   "max_abs_delta_raw_logits": 0.006176948547363281,
   "mean_abs_delta_raw_logits": 0.001166807604022324,
   "logits_min": -8.078125,
   "logits_max": 3.357421875,
   "raw_logits_min": -8.078125,
   "raw_logits_max": 3.357421875,
   "raw_logits_absmax": 8.078125,
   "oracle_logits_min": -8.074604034423828,
   "oracle_logits_max": 3.360363483428955,
   "oracle_raw_logits_absmax": 8.074604034423828,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5019691669149324,
   "seconds": 0.5019691669149324,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r07-team",
   "request_id": "r07",
   "question_id": "team",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 102,
   "nopts": 16,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.1630859375,
    -1.830078125,
    -2.40625,
    -1.9072265625,
    -2.966796875,
    -2.498046875,
    -2.65234375,
    -3.814453125,
    -3.65625,
    -3.5,
    -1.953125,
    4.99609375,
    -5.58984375,
    -4.71875,
    -4.734375,
    -4.8203125,
    -5.2109375,
    -4.5,
    -3.076171875,
    -3.603515625,
    -3.9296875,
    -1.462890625,
    -6.16796875,
    -6.16015625,
    -4.34765625,
    -5.6171875,
    -4.11328125,
    -5.16796875,
    -4.79296875,
    -7.55859375,
    -4.953125,
    -6.87890625,
    -6.35546875,
    -5.5390625,
    -3.66796875,
    -5.66796875,
    -5.40625,
    -5.59375,
    -4.90234375,
    -2.98828125,
    -5.25390625,
    -4.66796875,
    -4.9140625,
    -4.7109375,
    -4.44921875,
    -5.0078125,
    -4.515625,
    -4.4453125,
    -5.12109375,
    -4.80078125,
    -4.61328125,
    -5.0859375,
    -3.072265625,
    -5.03515625,
    -2.67578125,
    -2.328125,
    -4.07421875,
    -4.1953125,
    -5.2421875,
    -5.59765625,
    -7.96875,
    -6.88671875,
    -7.12109375,
    -5.91796875,
    -3.92578125,
    -6.97265625,
    -7.37109375,
    -2.31640625,
    -8.0078125,
    -4.48046875,
    -7.71484375,
    -5.30859375,
    -2.68359375,
    -5.828125,
    -3.888671875,
    -6.265625,
    -4.734375,
    -7.203125,
    -3.833984375,
    -5.74609375,
    -4.5,
    -4.20703125,
    -2.41015625,
    -4.265625,
    -3.494140625,
    -4.5703125,
    -2.09375,
    -5.14453125,
    -2.890625,
    -4.4609375,
    -3.953125,
    -3.95703125,
    -3.630859375,
    -4.25390625,
    -4.8515625,
    -4.5,
    -5.203125,
    -3.177734375,
    -3.69140625,
    -4.69140625,
    -5.984375,
    -4.9140625,
    -3.365234375,
    -4.18359375,
    -3.96484375,
    -6.0390625,
    -4.0625,
    -5.96875,
    -4.2421875,
    -6.07421875,
    -3.78515625,
    -4.9921875,
    -4.9296875,
    -6.3125,
    -5.41796875,
    -4.078125,
    -4.234375,
    -6.3828125,
    -4.67578125,
    -3.369140625,
    -2.8125,
    -3.736328125,
    -2.88671875,
    -3.48046875,
    -2.001953125,
    -5.875,
    -5.87890625,
    -3.20703125,
    -3.375,
    -5.296875,
    -5.32421875,
    -3.05859375,
    -5.57421875,
    -6.2578125,
    -5.04296875,
    -4.953125,
    -3.61328125,
    -5.3671875,
    -6.1015625,
    -3.384765625,
    -3.31640625,
    -4.01171875,
    -3.55859375,
    -2.58203125,
    -4.13671875,
    -5.04296875,
    -5.203125,
    -5.1484375,
    -5.4453125,
    -4.69921875,
    -5.17578125,
    -2.7421875,
    -1.66796875,
    -3.0625,
    -1.900390625,
    -1.796875,
    -0.94189453125,
    -1.736328125,
    -2.267578125,
    -0.5517578125,
    -1.759765625,
    -1.2353515625,
    -1.83203125,
    -1.1337890625,
    -2.97265625,
    -2.173828125,
    -3.107421875,
    -2.18359375,
    -3.955078125,
    -2.05078125,
    -2.244140625,
    -2.453125,
    -2.51953125,
    -2.91015625,
    -1.1572265625,
    -1.9599609375,
    -3.548828125,
    -3.66015625,
    -3.13671875,
    -3.314453125,
    -4.2578125,
    -3.3515625,
    -3.046875,
    -1.7802734375,
    -2.8046875,
    -2.3515625,
    -1.8134765625,
    -1.73828125,
    -2.58984375,
    -2.455078125,
    -1.74609375,
    -1.7939453125,
    -2.4296875,
    -1.4296875,
    -1.224609375,
    -2.185546875,
    -2.2109375,
    -2.513671875,
    -1.599609375,
    -1.287109375,
    -1.6181640625,
    -2.54296875,
    -2.146484375,
    -2.416015625,
    -2.265625,
    -1.6455078125,
    -2.12109375,
    -2.2265625,
    0.32763671875,
    -2.4140625,
    -1.50390625,
    -1.81640625,
    -0.33154296875,
    -1.57421875,
    -3.421875,
    -1.052734375,
    -2.48828125,
    -2.48046875,
    -0.98193359375,
    -2.400390625,
    -1.4755859375,
    -1.318359375,
    -1.130859375,
    -1.3154296875,
    0.018096923828125,
    0.1337890625,
    -0.405517578125,
    0.5068359375,
    0.921875,
    -0.182373046875,
    -1.064453125,
    -0.82568359375,
    0.12548828125,
    -0.463134765625,
    0.6201171875,
    0.76123046875,
    0.9541015625,
    -0.8525390625,
    0.2161865234375,
    -1.46875,
    0.215087890625,
    0.27978515625,
    -1.16015625,
    0.378662109375,
    -0.37548828125,
    0.8212890625,
    0.40673828125,
    0.27294921875,
    0.02471923828125,
    1.751953125,
    0.1199951171875,
    0.24609375,
    -0.2208251953125,
    0.70654296875,
    -0.19873046875,
    -3.7734375
   ],
   "raw256_argmax": 11,
   "option_argmax": 11,
   "oracle_argmax": 11,
   "argmax_agrees": true,
   "p_full": [
    0.002936795586720109,
    0.0015639415942132473,
    0.0009074636618606746,
    0.0014540127012878656,
    0.0005343779921531677,
    0.0008320833439938724,
    0.0007192206103354692,
    0.0002399235381744802,
    0.0002785992401186377,
    0.0003229134017601609,
    0.0013923137448728085,
    0.9882285594940186,
    4.484009696170688e-05,
    0.00010210772597929463,
    0.0001006115780910477,
    9.276608034269884e-05,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00024940224830061197
   ],
   "p": [
    0.0029375283047556877,
    0.0015643318183720112,
    0.000907690089661628,
    0.0014543754514306784,
    0.0005345113459043205,
    0.0008322909707203507,
    0.0007194000645540655,
    0.0002399834047537297,
    0.00027866874006576836,
    0.00032299396116286516,
    0.00139266112819314,
    0.9884751439094543,
    4.4851283746538684e-05,
    0.00010213319910690188,
    0.00010063668014481664,
    9.278922516386956e-05
   ],
   "p_oracle": [
    0.0029451660811901093,
    0.001555267721414566,
    0.0009126935619860888,
    0.0014587680343538523,
    0.0005313093424774706,
    0.000829861091915518,
    0.0007131857564672828,
    0.00023690734815318137,
    0.00027930724900215864,
    0.00032101283431984484,
    0.0013732233783230186,
    0.9885022044181824,
    4.6371802454814315e-05,
    0.00010244426084682345,
    0.000100217446743045,
    9.20594684430398e-05
   ],
   "abstain": 0.00024940224830061197,
   "abstain_oracle": 0.00024898501578718424,
   "abs_delta_abstain": 4.172325134277344e-07,
   "oracle_margin": 0.9855570197105408,
   "max_abs_delta_p": 2.7060508728027344e-05,
   "mean_abs_delta_p": 5.819915031679557e-06,
   "max_abs_delta_raw_logits": 0.03549671173095703,
   "mean_abs_delta_raw_logits": 0.002925039589172229,
   "logits_min": -8.0078125,
   "logits_max": 4.99609375,
   "raw_logits_min": -8.0078125,
   "raw_logits_max": 4.99609375,
   "raw_logits_absmax": 8.0078125,
   "oracle_logits_min": -8.008580207824707,
   "oracle_logits_max": 4.996329307556152,
   "oracle_raw_logits_absmax": 8.008580207824707,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.5688991250935942,
   "seconds": 0.5688991250935942,
   "full_vocab_argmax_id": 11,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r07-condition",
   "request_id": "r07",
   "question_id": "condition",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 43,
   "nopts": 4,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.359375,
    4.02734375,
    -2.384765625,
    -3.095703125,
    -2.283203125,
    -2.923828125,
    -2.810546875,
    -3.759765625,
    -3.6328125,
    -2.7265625,
    -3.380859375,
    -4.3359375,
    -5.609375,
    -3.51171875,
    -4.40625,
    -4.8359375,
    -3.7109375,
    -4.14453125,
    -2.919921875,
    -3.162109375,
    -3.35546875,
    -1.814453125,
    -5.15234375,
    -7.28125,
    -3.9921875,
    -5.57421875,
    -2.541015625,
    -5.76171875,
    -4.50390625,
    -7.1328125,
    -4.55859375,
    -5.0625,
    -6.45703125,
    -5.02734375,
    -3.01171875,
    -5.46875,
    -5.18359375,
    -5.7578125,
    -4.1484375,
    -3.09375,
    -7.50390625,
    -4.3671875,
    -4.85546875,
    -6.1484375,
    -4.71484375,
    -5.2734375,
    -4.59765625,
    -4.54296875,
    -5.3359375,
    -4.92578125,
    -6.0078125,
    -4.84375,
    -2.921875,
    -5.19140625,
    -2.787109375,
    -3.388671875,
    -3.6328125,
    -4.54296875,
    -4.8203125,
    -5.9375,
    -7.80078125,
    -6.640625,
    -7.8046875,
    -6.8359375,
    -3.787109375,
    -7.578125,
    -7.72265625,
    -2.330078125,
    -8.0078125,
    -4.2109375,
    -8.09375,
    -5.6015625,
    -2.75390625,
    -6.33984375,
    -3.73046875,
    -6.56640625,
    -4.296875,
    -6.9140625,
    -4.3984375,
    -5.953125,
    -4.32421875,
    -4.34375,
    -2.86328125,
    -4.20703125,
    -3.810546875,
    -3.9296875,
    -2.67578125,
    -3.982421875,
    -2.875,
    -5.2265625,
    -3.638671875,
    -4.28125,
    -4.41015625,
    -4.30859375,
    -4.6953125,
    -4.91015625,
    -5.2421875,
    -3.927734375,
    -4.1953125,
    -4.23828125,
    -6.40625,
    -4.68359375,
    -3.744140625,
    -4.375,
    -4.625,
    -6.4296875,
    -4.62890625,
    -5.87109375,
    -4.4296875,
    -5.6875,
    -4.46875,
    -4.9296875,
    -5.4375,
    -6.94921875,
    -4.88671875,
    -4.3046875,
    -4.87109375,
    -6.23828125,
    -4.33984375,
    -3.81640625,
    -3.55078125,
    -4.0625,
    -3.345703125,
    -3.83984375,
    -2.58984375,
    -6.01953125,
    -5.9453125,
    -3.3046875,
    -3.099609375,
    -5.71484375,
    -5.92578125,
    -3.927734375,
    -5.75390625,
    -6.09375,
    -5.19140625,
    -4.87890625,
    -4.1171875,
    -5.33203125,
    -5.9609375,
    -3.23828125,
    -3.81640625,
    -4.1953125,
    -3.93359375,
    -3.373046875,
    -5.0859375,
    -5.54296875,
    -6.3203125,
    -5.2421875,
    -4.765625,
    -4.6875,
    -4.87109375,
    -3.251953125,
    -1.1611328125,
    -3.681640625,
    -2.798828125,
    -1.6728515625,
    -2.22265625,
    -2.201171875,
    -3.548828125,
    -0.475830078125,
    -2.0703125,
    -2.064453125,
    -1.7255859375,
    -1.4541015625,
    -2.603515625,
    -2.51171875,
    -2.90625,
    -1.755859375,
    -3.345703125,
    -2.03515625,
    -2.345703125,
    -2.48828125,
    -2.6328125,
    -3.0703125,
    -0.93017578125,
    -2.076171875,
    -3.294921875,
    -3.427734375,
    -3.17578125,
    -3.619140625,
    -3.4296875,
    -3.4375,
    -2.8125,
    -2.068359375,
    -2.541015625,
    -1.5732421875,
    -1.3623046875,
    -1.9404296875,
    -1.8642578125,
    -2.265625,
    -1.0478515625,
    -1.8544921875,
    -3.095703125,
    -1.7666015625,
    -1.5634765625,
    -2.07421875,
    -2.072265625,
    -2.025390625,
    -1.587890625,
    -1.51171875,
    -1.2158203125,
    -2.0703125,
    -1.908203125,
    -1.880859375,
    -2.5234375,
    -1.0546875,
    -2.810546875,
    -2.720703125,
    -0.39404296875,
    -2.443359375,
    -1.005859375,
    -1.2939453125,
    -0.83642578125,
    -1.58203125,
    -2.375,
    -1.3330078125,
    -1.7763671875,
    -2.279296875,
    -1.83203125,
    -1.9658203125,
    -1.2138671875,
    -1.8759765625,
    -1.3828125,
    -1.515625,
    -0.420166015625,
    0.23681640625,
    -0.356689453125,
    0.95751953125,
    0.98486328125,
    -0.029449462890625,
    -0.94970703125,
    -0.54052734375,
    0.150634765625,
    -0.7646484375,
    0.5849609375,
    1.1630859375,
    1.4296875,
    -0.64306640625,
    0.037078857421875,
    -0.833984375,
    0.36669921875,
    0.763671875,
    -0.9482421875,
    -0.294677734375,
    0.214111328125,
    0.70751953125,
    0.3525390625,
    -0.386962890625,
    -0.96923828125,
    1.427734375,
    0.11614990234375,
    -0.8408203125,
    -0.144287109375,
    0.449462890625,
    -0.54736328125,
    -4.3515625
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.002381963422521949,
    0.9937416911125183,
    0.0023255073465406895,
    0.0011880503734573722,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003627287223935127
   ],
   "p": [
    0.002382827689871192,
    0.9941022992134094,
    0.002326351124793291,
    0.0011884814593940973
   ],
   "p_oracle": [
    0.0023597946856170893,
    0.9941675662994385,
    0.002299339510500431,
    0.0011732407147064805
   ],
   "abstain": 0.0003627287223935127,
   "abstain_oracle": 0.0003573091817088425,
   "abs_delta_abstain": 5.41954068467021e-06,
   "oracle_margin": 0.991807758808136,
   "max_abs_delta_p": 6.526708602905273e-05,
   "mean_abs_delta_p": 3.2638112315908074e-05,
   "max_abs_delta_raw_logits": 0.014191627502441406,
   "mean_abs_delta_raw_logits": 0.0047136940993368626,
   "logits_min": -8.09375,
   "logits_max": 4.02734375,
   "raw_logits_min": -8.09375,
   "raw_logits_max": 4.02734375,
   "raw_logits_absmax": 8.09375,
   "oracle_logits_min": -8.100370407104492,
   "oracle_logits_max": 4.035501003265381,
   "oracle_raw_logits_absmax": 8.100370407104492,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.24421899998560548,
   "seconds": 0.24421899998560548,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r07-refund",
   "request_id": "r07",
   "question_id": "refund",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 47,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    3.328125,
    -2.419921875,
    -2.2734375,
    -2.830078125,
    -2.353515625,
    -2.587890625,
    -2.751953125,
    -4.046875,
    -3.3046875,
    -2.54296875,
    -2.7578125,
    -3.30078125,
    -4.515625,
    -2.6484375,
    -4.109375,
    -4.3125,
    -4.41015625,
    -4.56640625,
    -2.57421875,
    -3.376953125,
    -3.443359375,
    -1.45703125,
    -4.92578125,
    -5.9609375,
    -3.798828125,
    -4.22265625,
    -3.0390625,
    -5.0859375,
    -4.13671875,
    -6.51171875,
    -4.09375,
    -5.03515625,
    -5.97265625,
    -4.30078125,
    -2.953125,
    -4.89453125,
    -5.09375,
    -6.09375,
    -4.21484375,
    -3.501953125,
    -6.36328125,
    -4.140625,
    -5.19140625,
    -6.15234375,
    -4.0859375,
    -4.8828125,
    -4.2421875,
    -4.85546875,
    -5.32421875,
    -4.0703125,
    -5.2890625,
    -4.25390625,
    -3.359375,
    -5.5078125,
    -3.06640625,
    -3.337890625,
    -4.13671875,
    -4.94921875,
    -4.75,
    -6.6796875,
    -7.390625,
    -6.98046875,
    -8.0234375,
    -5.8515625,
    -3.849609375,
    -7.28125,
    -7.77734375,
    -2.525390625,
    -7.51171875,
    -4.20703125,
    -8.1171875,
    -5.37890625,
    -3.412109375,
    -5.484375,
    -3.65625,
    -6.4609375,
    -4.24609375,
    -7.16015625,
    -3.9609375,
    -6.17578125,
    -4.81640625,
    -4.15625,
    -3.01171875,
    -4.66796875,
    -3.626953125,
    -4.59765625,
    -2.47265625,
    -4.07421875,
    -3.240234375,
    -4.85546875,
    -3.732421875,
    -4.35546875,
    -4.21875,
    -3.71875,
    -4.546875,
    -4.59375,
    -4.86328125,
    -3.51953125,
    -3.0625,
    -4.20703125,
    -5.70703125,
    -4.93359375,
    -3.390625,
    -3.970703125,
    -4.5625,
    -5.7890625,
    -4.0546875,
    -5.36328125,
    -4.4296875,
    -6.1796875,
    -4.66015625,
    -4.77734375,
    -5.265625,
    -6.16015625,
    -4.96484375,
    -3.888671875,
    -4.625,
    -5.9921875,
    -4.0,
    -3.9921875,
    -3.435546875,
    -4.4765625,
    -3.1640625,
    -3.7734375,
    -2.48828125,
    -5.4453125,
    -5.81640625,
    -2.83984375,
    -3.029296875,
    -5.70703125,
    -6.2265625,
    -2.853515625,
    -5.9140625,
    -6.4765625,
    -4.97265625,
    -3.943359375,
    -3.744140625,
    -4.84765625,
    -6.1484375,
    -2.896484375,
    -3.509765625,
    -4.390625,
    -3.48046875,
    -2.44921875,
    -5.03515625,
    -5.1953125,
    -5.91796875,
    -5.75,
    -5.25390625,
    -4.68359375,
    -4.671875,
    -2.765625,
    -1.091796875,
    -3.740234375,
    -3.03515625,
    -1.5654296875,
    -1.6845703125,
    -1.9228515625,
    -2.89453125,
    -0.517578125,
    -1.8671875,
    -2.134765625,
    -1.7060546875,
    -0.83544921875,
    -2.912109375,
    -1.8330078125,
    -2.70703125,
    -2.55078125,
    -3.87109375,
    -1.9443359375,
    -2.27734375,
    -2.611328125,
    -2.265625,
    -3.0546875,
    -0.75,
    -2.162109375,
    -3.90625,
    -3.40234375,
    -2.826171875,
    -2.291015625,
    -3.3515625,
    -2.650390625,
    -3.109375,
    -2.140625,
    -2.548828125,
    -1.7197265625,
    -1.4765625,
    -1.6396484375,
    -2.208984375,
    -2.8046875,
    -0.66650390625,
    -1.78125,
    -2.59765625,
    -1.552734375,
    -1.8701171875,
    -1.845703125,
    -2.279296875,
    -1.9482421875,
    -2.078125,
    -1.373046875,
    -1.4287109375,
    -2.033203125,
    -1.6826171875,
    -1.9755859375,
    -2.330078125,
    -1.087890625,
    -2.275390625,
    -2.42578125,
    0.2164306640625,
    -2.01171875,
    -1.4716796875,
    -1.052734375,
    -0.05419921875,
    -0.61279296875,
    -2.755859375,
    -0.99462890625,
    -1.6611328125,
    -1.9814453125,
    -1.1591796875,
    -1.7412109375,
    -1.01953125,
    -1.556640625,
    -0.599609375,
    -1.248046875,
    -0.314208984375,
    -0.170654296875,
    -0.60693359375,
    0.91357421875,
    0.8359375,
    -0.2120361328125,
    -0.467529296875,
    -0.66162109375,
    0.54345703125,
    -0.6884765625,
    0.3994140625,
    0.353759765625,
    1.18359375,
    -0.31689453125,
    0.1973876953125,
    -0.9013671875,
    0.495849609375,
    0.71142578125,
    -0.94091796875,
    -0.552734375,
    -0.09783935546875,
    0.99658203125,
    0.51318359375,
    -0.86962890625,
    -0.6630859375,
    2.328125,
    -0.10211181640625,
    -0.073974609375,
    -0.337158203125,
    0.345703125,
    -0.432861328125,
    -4.671875
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9963447451591492,
    0.0033025096636265516,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003527012886479497
   ],
   "p": [
    0.9966962933540344,
    0.0033036749809980392
   ],
   "p_oracle": [
    0.9967036843299866,
    0.0032962816767394543
   ],
   "abstain": 0.0003527012886479497,
   "abstain_oracle": 0.0003536264121066779,
   "abs_delta_abstain": 9.251234587281942e-07,
   "oracle_margin": 0.9934074282646179,
   "max_abs_delta_p": 7.393304258584976e-06,
   "mean_abs_delta_p": 7.392140105366707e-06,
   "max_abs_delta_raw_logits": 0.0057544708251953125,
   "mean_abs_delta_raw_logits": 0.0016469659167341888,
   "logits_min": -8.1171875,
   "logits_max": 3.328125,
   "raw_logits_min": -8.1171875,
   "raw_logits_max": 3.328125,
   "raw_logits_absmax": 8.1171875,
   "oracle_logits_min": -8.115945816040039,
   "oracle_logits_max": 3.3308520317077637,
   "oracle_raw_logits_absmax": 8.115945816040039,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2655132079962641,
   "seconds": 0.2655132079962641,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r08-bin",
   "request_id": "r08",
   "question_id": "bin",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 65,
   "nopts": 12,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.626953125,
    -1.9716796875,
    -2.66796875,
    -2.158203125,
    -1.857421875,
    -1.0283203125,
    6.43359375,
    -5.4140625,
    -2.357421875,
    -2.48828125,
    -3.765625,
    -4.44921875,
    -4.9453125,
    -2.029296875,
    -3.98046875,
    -5.203125,
    -4.9921875,
    -4.83984375,
    -2.21484375,
    -2.609375,
    -3.63671875,
    -1.6669921875,
    -5.3828125,
    -6.41015625,
    -4.71875,
    -4.890625,
    -2.388671875,
    -5.2265625,
    -4.72265625,
    -6.82421875,
    -5.29296875,
    -5.0625,
    -6.39453125,
    -4.66796875,
    -2.849609375,
    -4.79296875,
    -5.390625,
    -6.609375,
    -3.345703125,
    -4.01953125,
    -6.5234375,
    -4.14453125,
    -4.87890625,
    -5.359375,
    -4.64453125,
    -4.5234375,
    -5.4453125,
    -5.5,
    -5.4296875,
    -3.294921875,
    -5.73828125,
    -4.14453125,
    -1.546875,
    -5.69921875,
    -2.708984375,
    -3.576171875,
    -4.5078125,
    -5.4296875,
    -4.765625,
    -6.56640625,
    -7.734375,
    -6.48828125,
    -7.609375,
    -6.19921875,
    -3.759765625,
    -7.4375,
    -7.26171875,
    -2.96484375,
    -7.67578125,
    -3.7890625,
    -7.6796875,
    -4.859375,
    -2.841796875,
    -6.171875,
    -3.560546875,
    -5.875,
    -3.994140625,
    -6.796875,
    -3.748046875,
    -5.5703125,
    -4.87890625,
    -4.3125,
    -2.4453125,
    -4.1484375,
    -3.126953125,
    -4.51171875,
    -2.76953125,
    -4.04296875,
    -2.876953125,
    -4.5234375,
    -3.720703125,
    -4.953125,
    -4.83203125,
    -3.978515625,
    -5.37890625,
    -4.92578125,
    -5.27734375,
    -4.52734375,
    -4.16015625,
    -4.23046875,
    -5.65625,
    -4.68359375,
    -3.662109375,
    -4.39453125,
    -4.2734375,
    -5.51953125,
    -3.830078125,
    -5.41015625,
    -4.57421875,
    -5.56640625,
    -3.740234375,
    -4.5078125,
    -5.171875,
    -6.93359375,
    -5.07421875,
    -4.40625,
    -4.37890625,
    -5.35546875,
    -3.91796875,
    -3.181640625,
    -3.0390625,
    -4.59375,
    -3.412109375,
    -4.24609375,
    -2.892578125,
    -5.75,
    -5.82421875,
    -2.69140625,
    -3.072265625,
    -6.1875,
    -6.2421875,
    -2.833984375,
    -5.75390625,
    -6.515625,
    -5.53515625,
    -4.12109375,
    -3.943359375,
    -4.046875,
    -6.2421875,
    -3.298828125,
    -3.775390625,
    -3.576171875,
    -4.25,
    -2.80078125,
    -4.73046875,
    -5.51953125,
    -5.81640625,
    -5.34765625,
    -5.546875,
    -4.078125,
    -5.53125,
    -3.677734375,
    -0.82421875,
    -3.7890625,
    -2.513671875,
    -1.7568359375,
    -1.8876953125,
    -2.400390625,
    -3.693359375,
    -0.1920166015625,
    -1.43359375,
    -2.458984375,
    -1.666015625,
    -0.9052734375,
    -4.2109375,
    -2.369140625,
    -2.59375,
    -3.16796875,
    -3.013671875,
    -1.6806640625,
    -2.365234375,
    -2.4375,
    -2.236328125,
    -2.349609375,
    -1.0263671875,
    -2.21484375,
    -3.748046875,
    -3.380859375,
    -3.30078125,
    -2.935546875,
    -2.9296875,
    -2.86328125,
    -2.837890625,
    -2.28125,
    -3.150390625,
    -2.291015625,
    -2.279296875,
    -1.591796875,
    -1.951171875,
    -2.37890625,
    -0.87890625,
    -1.6455078125,
    -2.689453125,
    -2.048828125,
    -2.427734375,
    -1.99609375,
    -2.61328125,
    -1.5458984375,
    -2.05078125,
    -1.3603515625,
    -1.5615234375,
    -2.240234375,
    -2.697265625,
    -1.9951171875,
    -2.197265625,
    -1.17578125,
    -2.33203125,
    -1.9775390625,
    0.402099609375,
    -2.4921875,
    -0.72265625,
    -0.41552734375,
    -1.1064453125,
    -1.31640625,
    -2.98828125,
    -0.83203125,
    -2.40234375,
    -2.439453125,
    -1.3291015625,
    -1.484375,
    -0.7314453125,
    -1.828125,
    -1.357421875,
    -1.2041015625,
    -0.36962890625,
    0.0477294921875,
    -0.9052734375,
    1.1787109375,
    0.74951171875,
    -0.004180908203125,
    -0.7490234375,
    -0.432373046875,
    0.15478515625,
    -0.80712890625,
    0.703125,
    1.203125,
    1.7568359375,
    -0.9052734375,
    -0.334228515625,
    -0.95458984375,
    1.14453125,
    0.43896484375,
    -0.86474609375,
    -0.360107421875,
    -0.474609375,
    0.78173828125,
    0.03057861328125,
    -0.9111328125,
    -1.0615234375,
    1.2177734375,
    0.2012939453125,
    -0.404052734375,
    0.2081298828125,
    -0.032379150390625,
    -0.19970703125,
    -4.27734375
   ],
   "raw256_argmax": 6,
   "option_argmax": 6,
   "oracle_argmax": 6,
   "argmax_agrees": true,
   "p_full": [
    0.0004915074678137898,
    0.00035489167203195393,
    0.00018383270071353763,
    0.0002975564857479185,
    0.00039534244569949806,
    0.0008652399410493672,
    0.9967940449714661,
    1.3732745173911098e-05,
    0.00024650964769534767,
    0.00021784359705634415,
    6.517444853670895e-05,
    3.416748586460017e-05,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    4.0191087464336306e-05
   ],
   "p": [
    0.0004915272002108395,
    0.00035490593290887773,
    0.00018384009308647364,
    0.0002975684474222362,
    0.00039535833639092743,
    0.0008652746910229325,
    0.9968340992927551,
    1.3733297237195075e-05,
    0.00024651954299770296,
    0.0002178523427573964,
    6.517706788145006e-05,
    3.416885738261044e-05
   ],
   "p_oracle": [
    0.0004889668780378997,
    0.00035156647209078074,
    0.00018288858700543642,
    0.0002941089041996747,
    0.0003917153226211667,
    0.0008335465681739151,
    0.9968871474266052,
    1.3746554941462819e-05,
    0.0002432497713016346,
    0.00021473021479323506,
    6.454402318922803e-05,
    3.38076388288755e-05
   ],
   "abstain": 4.0191087464336306e-05,
   "abstain_oracle": 3.9258102333405986e-05,
   "abs_delta_abstain": 9.329851309303194e-07,
   "oracle_margin": 0.9960535764694214,
   "max_abs_delta_p": 5.3048133850097656e-05,
   "mean_abs_delta_p": 8.84412694783047e-06,
   "max_abs_delta_raw_logits": 0.02619314193725586,
   "mean_abs_delta_raw_logits": 0.0023197591362986714,
   "logits_min": -7.734375,
   "logits_max": 6.43359375,
   "raw_logits_min": -7.734375,
   "raw_logits_max": 6.43359375,
   "raw_logits_absmax": 7.734375,
   "oracle_logits_min": -7.736870288848877,
   "oracle_logits_max": 6.4470014572143555,
   "oracle_raw_logits_absmax": 7.736870288848877,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3651924169389531,
   "seconds": 0.3651924169389531,
   "full_vocab_argmax_id": 6,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r08-lamp",
   "request_id": "r08",
   "question_id": "lamp",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 54,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.376953125,
    3.7109375,
    -3.126953125,
    -2.68359375,
    -2.10546875,
    -2.642578125,
    -2.75,
    -3.908203125,
    -3.732421875,
    -2.9609375,
    -3.427734375,
    -4.2890625,
    -5.84375,
    -3.5859375,
    -4.61328125,
    -4.921875,
    -3.888671875,
    -4.16796875,
    -3.1171875,
    -3.263671875,
    -3.443359375,
    -1.7890625,
    -5.0546875,
    -7.375,
    -4.0625,
    -5.38671875,
    -2.55078125,
    -5.8828125,
    -4.56640625,
    -7.25390625,
    -4.75390625,
    -5.1015625,
    -6.671875,
    -5.33984375,
    -3.185546875,
    -5.7734375,
    -5.31640625,
    -6.0546875,
    -4.15234375,
    -3.22265625,
    -7.4921875,
    -4.3828125,
    -4.8046875,
    -6.0625,
    -4.73046875,
    -5.15234375,
    -4.73828125,
    -4.71484375,
    -5.19140625,
    -4.78125,
    -5.984375,
    -4.86328125,
    -2.73046875,
    -5.3125,
    -2.869140625,
    -3.166015625,
    -3.6328125,
    -4.66015625,
    -4.91015625,
    -6.18359375,
    -7.78515625,
    -6.765625,
    -7.88671875,
    -6.80078125,
    -3.716796875,
    -7.76171875,
    -7.83984375,
    -2.33203125,
    -8.0234375,
    -4.07421875,
    -8.3125,
    -5.7109375,
    -2.865234375,
    -6.36328125,
    -3.9453125,
    -6.6484375,
    -4.43359375,
    -7.015625,
    -4.19921875,
    -5.9921875,
    -4.35546875,
    -4.4140625,
    -2.853515625,
    -4.28515625,
    -3.705078125,
    -3.939453125,
    -2.751953125,
    -3.943359375,
    -2.88671875,
    -5.19921875,
    -3.703125,
    -4.29296875,
    -4.56640625,
    -4.1875,
    -4.7265625,
    -4.93359375,
    -5.3671875,
    -4.140625,
    -4.3203125,
    -4.375,
    -6.50390625,
    -4.93359375,
    -3.701171875,
    -4.38671875,
    -4.69140625,
    -6.48828125,
    -4.68359375,
    -5.90625,
    -4.51953125,
    -5.7109375,
    -4.3515625,
    -5.015625,
    -5.49609375,
    -7.0390625,
    -4.88671875,
    -4.44140625,
    -4.8046875,
    -6.27734375,
    -4.38671875,
    -3.6796875,
    -3.57421875,
    -3.96484375,
    -3.376953125,
    -4.0078125,
    -2.626953125,
    -6.203125,
    -6.09375,
    -3.298828125,
    -3.265625,
    -5.8125,
    -5.953125,
    -4.09375,
    -5.8046875,
    -6.23828125,
    -5.2109375,
    -4.91796875,
    -4.265625,
    -5.1484375,
    -6.125,
    -3.2265625,
    -3.939453125,
    -4.26171875,
    -4.1953125,
    -3.3046875,
    -5.25390625,
    -5.6328125,
    -6.41796875,
    -5.40625,
    -4.93359375,
    -4.75390625,
    -5.0390625,
    -3.466796875,
    -1.068359375,
    -3.703125,
    -2.927734375,
    -1.7451171875,
    -2.244140625,
    -2.1796875,
    -3.57421875,
    -0.4345703125,
    -2.20703125,
    -2.103515625,
    -1.712890625,
    -1.3037109375,
    -2.708984375,
    -2.626953125,
    -2.8984375,
    -1.9658203125,
    -3.388671875,
    -1.98828125,
    -2.419921875,
    -2.626953125,
    -2.748046875,
    -3.15234375,
    -1.0361328125,
    -1.9560546875,
    -3.333984375,
    -3.3125,
    -3.171875,
    -3.580078125,
    -3.556640625,
    -3.326171875,
    -2.900390625,
    -2.126953125,
    -2.607421875,
    -1.4892578125,
    -1.283203125,
    -1.888671875,
    -1.814453125,
    -2.2265625,
    -1.048828125,
    -1.7080078125,
    -3.119140625,
    -1.8515625,
    -1.50390625,
    -2.09765625,
    -2.21875,
    -2.080078125,
    -1.814453125,
    -1.37890625,
    -1.1689453125,
    -2.01171875,
    -1.861328125,
    -1.92578125,
    -2.421875,
    -1.154296875,
    -2.845703125,
    -2.67578125,
    -0.370361328125,
    -2.58203125,
    -0.83740234375,
    -1.2861328125,
    -0.81005859375,
    -1.6083984375,
    -2.421875,
    -1.34765625,
    -1.9208984375,
    -2.400390625,
    -1.869140625,
    -2.025390625,
    -1.2763671875,
    -1.697265625,
    -1.3818359375,
    -1.556640625,
    -0.2431640625,
    0.142578125,
    -0.26123046875,
    1.0517578125,
    0.95556640625,
    -0.05908203125,
    -0.96435546875,
    -0.62841796875,
    0.27783203125,
    -0.7890625,
    0.57763671875,
    1.0771484375,
    1.4892578125,
    -0.6015625,
    0.075927734375,
    -0.74072265625,
    0.329345703125,
    0.78857421875,
    -0.71728515625,
    -0.38427734375,
    0.07177734375,
    0.74365234375,
    0.3154296875,
    -0.302978515625,
    -0.9375,
    1.5966796875,
    0.11029052734375,
    -0.70166015625,
    -0.1976318359375,
    0.60107421875,
    -0.429931640625,
    -4.29296875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0031621637754142284,
    0.9947633743286133,
    0.0015569502720609307,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0005174673860892653
   ],
   "p": [
    0.003163801273331046,
    0.9952784776687622,
    0.0015577564481645823
   ],
   "p_oracle": [
    0.0031704173889011145,
    0.9952751398086548,
    0.0015544575871899724
   ],
   "abstain": 0.0005174673860892653,
   "abstain_oracle": 0.0005142955924384296,
   "abs_delta_abstain": 3.1717936508357525e-06,
   "oracle_margin": 0.992104709148407,
   "max_abs_delta_p": 6.616115570068359e-06,
   "mean_abs_delta_p": 4.417612217366695e-06,
   "max_abs_delta_raw_logits": 0.011770486831665039,
   "mean_abs_delta_raw_logits": 0.0029316477885004133,
   "logits_min": -8.3125,
   "logits_max": 3.7109375,
   "raw_logits_min": -8.3125,
   "raw_logits_max": 3.7109375,
   "raw_logits_absmax": 8.3125,
   "oracle_logits_min": -8.319535255432129,
   "oracle_logits_max": 3.7114009857177734,
   "oracle_raw_logits_absmax": 8.319535255432129,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.304458332946524,
   "seconds": 0.304458332946524,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r08-closed",
   "request_id": "r08",
   "question_id": "closed",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 36,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -3.466796875,
    1.412109375,
    -1.361328125,
    -1.4951171875,
    -1.091796875,
    -2.55859375,
    -1.7275390625,
    -2.7578125,
    -3.095703125,
    -1.5849609375,
    -2.328125,
    -3.68359375,
    -4.43359375,
    -2.1953125,
    -2.9921875,
    -4.20703125,
    -3.798828125,
    -4.59375,
    -2.66015625,
    -2.41015625,
    -3.37109375,
    -1.5673828125,
    -4.9453125,
    -6.46875,
    -3.828125,
    -4.578125,
    -2.5703125,
    -4.90234375,
    -3.787109375,
    -5.609375,
    -4.19921875,
    -3.849609375,
    -5.73046875,
    -4.3359375,
    -2.419921875,
    -4.84765625,
    -5.015625,
    -5.6015625,
    -4.0234375,
    -2.97265625,
    -6.88671875,
    -4.03515625,
    -4.5078125,
    -5.89453125,
    -3.58984375,
    -4.54296875,
    -4.25,
    -4.44921875,
    -4.83984375,
    -4.1484375,
    -5.265625,
    -4.30859375,
    -2.556640625,
    -4.69921875,
    -2.40234375,
    -3.208984375,
    -3.341796875,
    -4.546875,
    -4.28125,
    -5.10546875,
    -6.96875,
    -5.9921875,
    -7.19921875,
    -6.06640625,
    -3.09375,
    -7.18359375,
    -7.23046875,
    -2.27734375,
    -7.51171875,
    -4.13671875,
    -7.54296875,
    -5.078125,
    -2.25,
    -5.60546875,
    -3.603515625,
    -5.79296875,
    -4.1015625,
    -6.23046875,
    -3.822265625,
    -5.48046875,
    -4.12890625,
    -3.767578125,
    -2.552734375,
    -3.837890625,
    -3.09765625,
    -3.984375,
    -2.41015625,
    -3.3046875,
    -2.7734375,
    -4.34765625,
    -3.271484375,
    -3.87890625,
    -3.46875,
    -4.0078125,
    -4.4921875,
    -4.7109375,
    -4.6875,
    -3.59375,
    -3.53125,
    -4.046875,
    -5.3046875,
    -4.53515625,
    -3.302734375,
    -3.568359375,
    -4.3125,
    -5.484375,
    -3.572265625,
    -5.15234375,
    -3.916015625,
    -4.98828125,
    -4.2109375,
    -4.2890625,
    -4.52734375,
    -5.9921875,
    -4.33984375,
    -3.599609375,
    -4.41796875,
    -5.41015625,
    -4.11328125,
    -3.0859375,
    -3.146484375,
    -3.947265625,
    -3.12890625,
    -3.521484375,
    -2.435546875,
    -5.39453125,
    -4.9296875,
    -2.86328125,
    -2.90625,
    -4.99609375,
    -5.38671875,
    -2.880859375,
    -5.46875,
    -5.7265625,
    -4.890625,
    -4.23046875,
    -3.51953125,
    -4.484375,
    -5.53125,
    -2.4140625,
    -3.275390625,
    -3.90625,
    -3.546875,
    -3.306640625,
    -4.37109375,
    -4.66015625,
    -5.37890625,
    -4.81640625,
    -4.88671875,
    -4.09765625,
    -4.46875,
    -3.03125,
    -0.8447265625,
    -3.017578125,
    -2.275390625,
    -1.3046875,
    -1.62109375,
    -1.8720703125,
    -3.142578125,
    -0.453125,
    -1.9521484375,
    -1.91796875,
    -1.2197265625,
    -1.185546875,
    -2.63671875,
    -2.24609375,
    -2.43359375,
    -1.4736328125,
    -3.13671875,
    -1.8916015625,
    -1.6748046875,
    -2.31640625,
    -1.947265625,
    -2.607421875,
    -0.74072265625,
    -1.5830078125,
    -2.845703125,
    -3.158203125,
    -2.603515625,
    -3.107421875,
    -2.859375,
    -2.55078125,
    -2.6875,
    -1.7568359375,
    -2.201171875,
    -1.328125,
    -1.447265625,
    -1.509765625,
    -1.4599609375,
    -1.8779296875,
    -0.5263671875,
    -1.443359375,
    -2.48828125,
    -1.3369140625,
    -1.400390625,
    -1.869140625,
    -1.8564453125,
    -1.48046875,
    -1.3193359375,
    -1.0537109375,
    -1.2041015625,
    -1.474609375,
    -1.3798828125,
    -1.4609375,
    -2.029296875,
    -0.59228515625,
    -2.390625,
    -2.2265625,
    -0.06085205078125,
    -1.7841796875,
    -0.544921875,
    -0.9677734375,
    -0.513671875,
    -0.81103515625,
    -2.296875,
    -0.86376953125,
    -1.5556640625,
    -2.146484375,
    -1.5888671875,
    -1.5390625,
    -1.251953125,
    -1.5712890625,
    -1.46484375,
    -1.24609375,
    -0.1876220703125,
    0.3232421875,
    -0.2166748046875,
    1.076171875,
    0.990234375,
    -0.022705078125,
    -0.712890625,
    -0.451416015625,
    -0.07757568359375,
    -0.76904296875,
    0.5234375,
    1.2822265625,
    1.671875,
    -0.78564453125,
    -0.2496337890625,
    -0.7529296875,
    0.81494140625,
    0.775390625,
    -0.544921875,
    -0.1353759765625,
    -0.0765380859375,
    0.517578125,
    0.2471923828125,
    -0.6767578125,
    -0.99951171875,
    1.4951171875,
    0.0089263916015625,
    -0.89453125,
    -0.0823974609375,
    0.60791015625,
    -0.5576171875,
    -6.66015625
   ],
   "raw256_argmax": 236,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.00779504282400012,
    0.9918781518936157,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003268002765253186
   ],
   "p": [
    0.007797591388225555,
    0.9922024011611938
   ],
   "p_oracle": [
    0.007764671463519335,
    0.9922353625297546
   ],
   "abstain": 0.0003268002765253186,
   "abstain_oracle": 0.0003250511654186994,
   "abs_delta_abstain": 1.7491111066192389e-06,
   "oracle_margin": 0.9844706654548645,
   "max_abs_delta_p": 3.2961368560791016e-05,
   "mean_abs_delta_p": 3.294064663350582e-05,
   "max_abs_delta_raw_logits": 0.006103038787841797,
   "mean_abs_delta_raw_logits": 0.0011034125345759094,
   "logits_min": -7.54296875,
   "logits_max": 1.671875,
   "raw_logits_min": -7.54296875,
   "raw_logits_max": 1.671875,
   "raw_logits_absmax": 7.54296875,
   "oracle_logits_min": -7.544649600982666,
   "oracle_logits_max": 1.6727880239486694,
   "oracle_raw_logits_absmax": 7.544649600982666,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2114128340035677,
   "seconds": 0.2114128340035677,
   "full_vocab_argmax_id": 236,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r09-liquid",
   "request_id": "r09",
   "question_id": "liquid",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 46,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -3.546875,
    2.56640625,
    -2.720703125,
    -3.09765625,
    -2.9921875,
    -3.392578125,
    -3.09765625,
    -3.72265625,
    -3.978515625,
    -2.8828125,
    -3.4453125,
    -3.80078125,
    -5.0703125,
    -3.041015625,
    -4.1875,
    -4.421875,
    -4.02734375,
    -4.08203125,
    -3.068359375,
    -3.341796875,
    -3.697265625,
    -2.38671875,
    -5.484375,
    -6.8984375,
    -3.962890625,
    -5.65625,
    -2.587890625,
    -5.265625,
    -4.75,
    -6.625,
    -4.7734375,
    -4.58984375,
    -6.3359375,
    -4.953125,
    -3.623046875,
    -5.5546875,
    -5.703125,
    -6.0,
    -4.46484375,
    -3.1875,
    -7.23828125,
    -4.93359375,
    -4.79296875,
    -5.921875,
    -4.64453125,
    -5.375,
    -4.51171875,
    -4.2109375,
    -5.671875,
    -4.73828125,
    -5.6953125,
    -5.0,
    -2.99609375,
    -4.7890625,
    -3.01953125,
    -3.529296875,
    -3.55859375,
    -4.06640625,
    -4.91796875,
    -5.109375,
    -7.546875,
    -6.3984375,
    -7.671875,
    -6.7734375,
    -3.720703125,
    -7.38671875,
    -7.7734375,
    -2.630859375,
    -8.125,
    -4.4765625,
    -7.71484375,
    -5.1640625,
    -2.294921875,
    -6.25,
    -3.890625,
    -6.21484375,
    -4.2890625,
    -6.7265625,
    -4.34765625,
    -5.7265625,
    -4.40234375,
    -4.06640625,
    -2.814453125,
    -4.48046875,
    -3.65625,
    -4.0546875,
    -2.55859375,
    -3.814453125,
    -3.296875,
    -4.87890625,
    -3.826171875,
    -4.20703125,
    -4.453125,
    -4.33984375,
    -4.5625,
    -4.59375,
    -5.23828125,
    -3.90234375,
    -4.17578125,
    -4.25390625,
    -6.26953125,
    -4.30078125,
    -3.947265625,
    -4.3515625,
    -4.5546875,
    -6.203125,
    -4.34375,
    -5.77734375,
    -3.951171875,
    -5.765625,
    -4.41796875,
    -4.7734375,
    -5.29296875,
    -6.33984375,
    -5.04296875,
    -3.94140625,
    -4.7578125,
    -6.45703125,
    -4.78125,
    -3.525390625,
    -3.337890625,
    -4.03125,
    -3.33984375,
    -3.833984375,
    -2.44140625,
    -5.98828125,
    -5.9609375,
    -3.212890625,
    -3.3671875,
    -5.66015625,
    -5.48828125,
    -4.01171875,
    -5.71875,
    -6.1484375,
    -4.859375,
    -5.13671875,
    -3.982421875,
    -4.89453125,
    -6.1171875,
    -3.078125,
    -3.7109375,
    -4.20703125,
    -3.57421875,
    -3.615234375,
    -4.5,
    -5.12109375,
    -6.28515625,
    -5.24609375,
    -4.5703125,
    -4.265625,
    -4.953125,
    -3.294921875,
    -1.484375,
    -3.6171875,
    -2.244140625,
    -1.6787109375,
    -1.9931640625,
    -1.9892578125,
    -3.361328125,
    -0.75537109375,
    -2.357421875,
    -2.029296875,
    -1.9130859375,
    -1.611328125,
    -2.53125,
    -2.33203125,
    -3.06640625,
    -1.8720703125,
    -3.619140625,
    -2.373046875,
    -2.462890625,
    -2.552734375,
    -2.294921875,
    -3.083984375,
    -0.9775390625,
    -2.173828125,
    -3.337890625,
    -3.55859375,
    -3.55078125,
    -3.8046875,
    -3.8359375,
    -3.50390625,
    -2.76953125,
    -2.490234375,
    -2.84765625,
    -2.005859375,
    -1.5234375,
    -2.0859375,
    -2.294921875,
    -2.27734375,
    -1.4248046875,
    -2.05859375,
    -3.46875,
    -2.1015625,
    -1.583984375,
    -2.5546875,
    -2.244140625,
    -2.14453125,
    -1.7890625,
    -1.611328125,
    -1.6708984375,
    -2.251953125,
    -1.8359375,
    -2.140625,
    -2.412109375,
    -1.5048828125,
    -2.736328125,
    -2.537109375,
    -0.43359375,
    -2.416015625,
    -1.1396484375,
    -1.4384765625,
    -1.3154296875,
    -1.6533203125,
    -2.896484375,
    -1.681640625,
    -1.935546875,
    -2.51953125,
    -2.083984375,
    -2.19921875,
    -1.7509765625,
    -1.5185546875,
    -1.4755859375,
    -1.3662109375,
    -0.17333984375,
    0.10986328125,
    -0.64208984375,
    0.6767578125,
    1.1103515625,
    0.00887298583984375,
    -0.62744140625,
    -0.452880859375,
    -0.12225341796875,
    -0.52685546875,
    0.384033203125,
    1.5078125,
    1.287109375,
    -0.6787109375,
    -0.04339599609375,
    -0.9716796875,
    0.291015625,
    0.5703125,
    -0.9609375,
    -0.10980224609375,
    -0.093994140625,
    0.398681640625,
    0.576171875,
    -0.158447265625,
    -0.8193359375,
    1.2958984375,
    -0.443603515625,
    -0.5859375,
    -0.3408203125,
    0.447998046875,
    -0.436767578125,
    -2.380859375
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0030449600890278816,
    0.9811477065086365,
    0.006645732559263706,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.009161652065813541
   ],
   "p": [
    0.0030731146689504385,
    0.9902197122573853,
    0.0067071812227368355
   ],
   "p_oracle": [
    0.003028564853593707,
    0.9903112053871155,
    0.006660241633653641
   ],
   "abstain": 0.009161652065813541,
   "abstain_oracle": 0.009437600150704384,
   "abs_delta_abstain": 0.00027594808489084244,
   "oracle_margin": 0.9836509823799133,
   "max_abs_delta_p": 9.149312973022461e-05,
   "mean_abs_delta_p": 6.099417805671692e-05,
   "max_abs_delta_raw_logits": 0.02923440933227539,
   "mean_abs_delta_raw_logits": 0.005249841371551156,
   "logits_min": -8.125,
   "logits_max": 2.56640625,
   "raw_logits_min": -8.125,
   "raw_logits_max": 2.56640625,
   "raw_logits_absmax": 8.125,
   "oracle_logits_min": -8.139936447143555,
   "oracle_logits_max": 2.5640313625335693,
   "oracle_raw_logits_absmax": 8.139936447143555,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2644187909318134,
   "seconds": 0.2644187909318134,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r09-valve",
   "request_id": "r09",
   "question_id": "valve",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 51,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    3.373046875,
    -2.31640625,
    -2.28125,
    -2.71484375,
    -2.38671875,
    -2.59765625,
    -3.0234375,
    -4.0859375,
    -2.98828125,
    -2.47265625,
    -2.80859375,
    -3.509765625,
    -4.6171875,
    -2.568359375,
    -4.0546875,
    -4.3671875,
    -4.3515625,
    -4.578125,
    -2.724609375,
    -3.5,
    -3.388671875,
    -1.427734375,
    -5.01171875,
    -6.2578125,
    -3.818359375,
    -4.48046875,
    -2.9453125,
    -5.32421875,
    -4.15234375,
    -6.640625,
    -4.26171875,
    -5.3984375,
    -6.1875,
    -4.49609375,
    -3.099609375,
    -5.0078125,
    -5.1484375,
    -6.22265625,
    -4.31640625,
    -3.6875,
    -6.671875,
    -4.46484375,
    -5.26953125,
    -6.1796875,
    -4.28125,
    -4.8515625,
    -4.1953125,
    -4.93359375,
    -5.19921875,
    -4.1171875,
    -5.55078125,
    -4.14453125,
    -3.3046875,
    -5.60546875,
    -3.15625,
    -3.373046875,
    -4.14453125,
    -4.98828125,
    -4.9140625,
    -6.70703125,
    -7.5390625,
    -7.171875,
    -8.1328125,
    -6.015625,
    -3.916015625,
    -7.359375,
    -7.76953125,
    -2.3671875,
    -7.60546875,
    -4.22265625,
    -8.375,
    -5.41015625,
    -3.46484375,
    -5.64453125,
    -3.810546875,
    -6.578125,
    -4.40234375,
    -7.30859375,
    -3.9765625,
    -6.109375,
    -4.80078125,
    -4.140625,
    -3.197265625,
    -4.60546875,
    -3.63671875,
    -4.4375,
    -2.654296875,
    -4.16015625,
    -3.2421875,
    -4.9140625,
    -3.87890625,
    -4.484375,
    -4.30859375,
    -3.705078125,
    -4.57421875,
    -4.734375,
    -4.98828125,
    -3.560546875,
    -3.275390625,
    -4.2734375,
    -5.8203125,
    -5.06640625,
    -3.34765625,
    -4.2421875,
    -4.64453125,
    -5.91796875,
    -4.13671875,
    -5.33203125,
    -4.5625,
    -6.15625,
    -4.75,
    -4.921875,
    -5.328125,
    -6.3203125,
    -5.16796875,
    -4.08203125,
    -4.72265625,
    -6.0078125,
    -4.0078125,
    -4.02734375,
    -3.365234375,
    -4.453125,
    -3.2578125,
    -3.92578125,
    -2.423828125,
    -5.51953125,
    -5.8984375,
    -2.921875,
    -2.951171875,
    -5.89453125,
    -6.30078125,
    -2.87890625,
    -5.921875,
    -6.56640625,
    -5.09765625,
    -4.0234375,
    -3.865234375,
    -4.9921875,
    -5.98828125,
    -2.982421875,
    -3.529296875,
    -4.41015625,
    -3.46875,
    -2.388671875,
    -5.171875,
    -5.2421875,
    -6.0546875,
    -5.74609375,
    -5.3125,
    -4.89453125,
    -4.59765625,
    -2.9296875,
    -1.259765625,
    -3.845703125,
    -3.1015625,
    -1.6005859375,
    -1.947265625,
    -2.015625,
    -3.01171875,
    -0.35595703125,
    -1.8359375,
    -2.212890625,
    -1.775390625,
    -0.9111328125,
    -2.990234375,
    -1.9052734375,
    -2.740234375,
    -2.607421875,
    -3.86328125,
    -1.951171875,
    -2.45703125,
    -2.623046875,
    -2.34765625,
    -3.13671875,
    -0.888671875,
    -2.16015625,
    -3.8359375,
    -3.41015625,
    -2.958984375,
    -2.435546875,
    -3.275390625,
    -2.62890625,
    -3.103515625,
    -2.037109375,
    -2.595703125,
    -1.84765625,
    -1.4482421875,
    -1.517578125,
    -2.111328125,
    -2.849609375,
    -0.71728515625,
    -1.734375,
    -2.626953125,
    -1.6103515625,
    -1.919921875,
    -2.04296875,
    -2.2734375,
    -2.10546875,
    -2.28125,
    -1.443359375,
    -1.3515625,
    -2.09765625,
    -1.8291015625,
    -2.076171875,
    -2.498046875,
    -1.1142578125,
    -2.2265625,
    -2.6171875,
    0.275634765625,
    -2.109375,
    -1.462890625,
    -1.0908203125,
    -0.1717529296875,
    -0.720703125,
    -2.6171875,
    -1.02734375,
    -1.693359375,
    -1.970703125,
    -1.2626953125,
    -1.7841796875,
    -0.98828125,
    -1.7197265625,
    -0.57177734375,
    -1.1708984375,
    -0.380859375,
    -0.28271484375,
    -0.419677734375,
    0.86669921875,
    0.8916015625,
    -0.1614990234375,
    -0.42333984375,
    -0.6875,
    0.634765625,
    -0.68017578125,
    0.39013671875,
    0.38330078125,
    1.173828125,
    -0.34814453125,
    0.220947265625,
    -0.7509765625,
    0.286865234375,
    0.78955078125,
    -1.0302734375,
    -0.457275390625,
    -0.2215576171875,
    1.0751953125,
    0.48876953125,
    -0.8037109375,
    -0.66455078125,
    2.31640625,
    0.00951385498046875,
    -0.026153564453125,
    -0.3310546875,
    0.3056640625,
    -0.44140625,
    -4.6171875
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.99614417552948,
    0.0034997137263417244,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003560677287168801
   ],
   "p": [
    0.9964990615844727,
    0.003500960534438491
   ],
   "p_oracle": [
    0.9965094327926636,
    0.0034905539359897375
   ],
   "abstain": 0.0003560677287168801,
   "abstain_oracle": 0.0003569536202121526,
   "abs_delta_abstain": 8.858914952725172e-07,
   "oracle_margin": 0.9930188655853271,
   "max_abs_delta_p": 1.0406598448753357e-05,
   "mean_abs_delta_p": 1.0388903319835663e-05,
   "max_abs_delta_raw_logits": 0.007826805114746094,
   "mean_abs_delta_raw_logits": 0.0027350594173185527,
   "logits_min": -8.375,
   "logits_max": 3.373046875,
   "raw_logits_min": -8.375,
   "raw_logits_max": 3.373046875,
   "raw_logits_absmax": 8.375,
   "oracle_logits_min": -8.38168716430664,
   "oracle_logits_max": 3.3759267330169678,
   "oracle_raw_logits_absmax": 8.38168716430664,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2914992499863729,
   "seconds": 0.2914992499863729,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r09-fill",
   "request_id": "r09",
   "question_id": "fill",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 72,
   "nopts": 5,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    -1.87890625,
    -0.270751953125,
    4.1328125,
    -2.048828125,
    -2.96484375,
    -2.49609375,
    -2.529296875,
    -3.669921875,
    -2.48046875,
    -2.189453125,
    -3.138671875,
    -3.771484375,
    -4.828125,
    -3.177734375,
    -3.62890625,
    -4.49609375,
    -3.810546875,
    -4.40625,
    -2.447265625,
    -3.12109375,
    -3.787109375,
    -1.2431640625,
    -5.00390625,
    -5.83984375,
    -2.935546875,
    -4.41796875,
    -2.814453125,
    -4.01953125,
    -3.814453125,
    -6.5234375,
    -4.27734375,
    -6.11328125,
    -4.71484375,
    -4.26953125,
    -2.763671875,
    -4.9609375,
    -5.20703125,
    -6.70703125,
    -4.69140625,
    -3.361328125,
    -6.2578125,
    -4.15234375,
    -4.55859375,
    -6.046875,
    -4.72265625,
    -5.03125,
    -4.49609375,
    -4.22265625,
    -5.2265625,
    -5.04296875,
    -5.30078125,
    -4.9921875,
    -2.91796875,
    -5.0078125,
    -3.091796875,
    -3.68359375,
    -4.01171875,
    -5.125,
    -5.0390625,
    -6.58203125,
    -7.453125,
    -6.72265625,
    -7.50390625,
    -6.109375,
    -3.31640625,
    -6.9453125,
    -7.484375,
    -2.833984375,
    -7.84375,
    -4.75390625,
    -7.7265625,
    -5.35546875,
    -2.923828125,
    -5.80078125,
    -2.80859375,
    -6.2109375,
    -3.7890625,
    -6.64453125,
    -4.4453125,
    -5.69921875,
    -4.578125,
    -3.623046875,
    -2.529296875,
    -4.38671875,
    -3.41015625,
    -4.38671875,
    -2.642578125,
    -4.12890625,
    -2.89453125,
    -4.6640625,
    -3.619140625,
    -3.91796875,
    -4.3671875,
    -3.515625,
    -4.9140625,
    -4.7421875,
    -5.109375,
    -3.5078125,
    -3.412109375,
    -4.359375,
    -5.6171875,
    -4.37109375,
    -3.828125,
    -3.638671875,
    -4.21484375,
    -5.75390625,
    -3.744140625,
    -5.34765625,
    -4.46875,
    -5.66796875,
    -4.15234375,
    -4.2890625,
    -5.47265625,
    -6.10546875,
    -4.625,
    -3.853515625,
    -4.546875,
    -5.921875,
    -3.619140625,
    -3.384765625,
    -3.638671875,
    -4.359375,
    -3.232421875,
    -3.02734375,
    -2.52734375,
    -5.38671875,
    -5.54296875,
    -3.27734375,
    -2.482421875,
    -5.37109375,
    -5.38671875,
    -2.56640625,
    -5.4921875,
    -5.92578125,
    -5.12109375,
    -4.68359375,
    -3.611328125,
    -4.63671875,
    -5.65234375,
    -3.55078125,
    -3.224609375,
    -4.015625,
    -3.546875,
    -2.80078125,
    -4.53515625,
    -4.6875,
    -5.5390625,
    -4.9765625,
    -4.83984375,
    -4.40625,
    -4.71484375,
    -2.96875,
    -1.01171875,
    -3.716796875,
    -2.369140625,
    -1.1064453125,
    -1.916015625,
    -2.087890625,
    -2.98046875,
    0.0093994140625,
    -1.736328125,
    -1.8681640625,
    -1.53515625,
    -1.5126953125,
    -3.3515625,
    -2.208984375,
    -3.130859375,
    -1.8740234375,
    -3.49609375,
    -2.029296875,
    -2.125,
    -2.345703125,
    -2.216796875,
    -2.68359375,
    -0.681640625,
    -1.9296875,
    -2.70703125,
    -3.142578125,
    -2.654296875,
    -2.84375,
    -2.654296875,
    -2.87109375,
    -2.8515625,
    -1.9951171875,
    -2.578125,
    -1.462890625,
    -2.0390625,
    -1.3818359375,
    -1.6435546875,
    -2.83984375,
    -0.7529296875,
    -1.7265625,
    -2.728515625,
    -1.5595703125,
    -1.8701171875,
    -2.19921875,
    -1.677734375,
    -1.8056640625,
    -1.6064453125,
    -1.31640625,
    -1.1396484375,
    -1.8935546875,
    -2.05078125,
    -1.853515625,
    -2.75,
    -0.59619140625,
    -2.619140625,
    -2.40625,
    0.02655029296875,
    -2.185546875,
    -0.8828125,
    -1.0751953125,
    -0.2454833984375,
    -1.6123046875,
    -2.966796875,
    -0.9970703125,
    -1.5546875,
    -2.2265625,
    -1.34765625,
    -1.6689453125,
    -0.6064453125,
    -2.236328125,
    -1.162109375,
    -1.6787109375,
    -0.00858306884765625,
    0.1539306640625,
    -0.1866455078125,
    1.3125,
    0.890625,
    -0.354736328125,
    -0.388427734375,
    -0.728515625,
    0.2265625,
    -0.87451171875,
    0.8115234375,
    0.78759765625,
    1.341796875,
    -0.449951171875,
    0.10247802734375,
    -1.0517578125,
    0.6513671875,
    0.4609375,
    -0.6826171875,
    -0.042816162109375,
    -0.11083984375,
    1.2021484375,
    0.61767578125,
    -0.79541015625,
    -0.87890625,
    1.9345703125,
    -0.30908203125,
    -0.6953125,
    0.2362060546875,
    0.5595703125,
    0.051849365234375,
    -5.80859375
   ],
   "raw256_argmax": 2,
   "option_argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "p_full": [
    0.0030747512355446815,
    0.014366081915795803,
    0.9787898659706116,
    0.002612557727843523,
    0.0010856700828298926,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    7.10803724359721e-05
   ],
   "p": [
    0.0030749698635190725,
    0.014367103576660156,
    0.9788594841957092,
    0.002612743526697159,
    0.001085747266188264
   ],
   "p_oracle": [
    0.0030626419465988874,
    0.014278288930654526,
    0.9789782762527466,
    0.002598386723548174,
    0.0010823768097907305
   ],
   "abstain": 7.10803724359721e-05,
   "abstain_oracle": 7.085428660502657e-05,
   "abs_delta_abstain": 2.260858309455216e-07,
   "oracle_margin": 0.9646999835968018,
   "max_abs_delta_p": 0.00011879205703735352,
   "mean_abs_delta_p": 4.7532375901937485e-05,
   "max_abs_delta_raw_logits": 0.006778240203857422,
   "mean_abs_delta_raw_logits": 0.0012882460723631084,
   "logits_min": -7.84375,
   "logits_max": 4.1328125,
   "raw_logits_min": -7.84375,
   "raw_logits_max": 4.1328125,
   "raw_logits_absmax": 7.84375,
   "oracle_logits_min": -7.844677448272705,
   "oracle_logits_max": 4.1366448402404785,
   "oracle_raw_logits_absmax": 7.844677448272705,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.40510362503118813,
   "seconds": 0.40510362503118813,
   "full_vocab_argmax_id": 2,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r10-surface",
   "request_id": "r10",
   "question_id": "surface",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 45,
   "nopts": 4,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.6796875,
    4.3515625,
    -2.443359375,
    -3.22265625,
    -2.14453125,
    -2.771484375,
    -2.87109375,
    -3.72265625,
    -3.681640625,
    -2.748046875,
    -3.517578125,
    -4.09765625,
    -5.671875,
    -3.828125,
    -4.71484375,
    -4.91015625,
    -3.693359375,
    -4.07421875,
    -3.099609375,
    -3.201171875,
    -3.365234375,
    -1.8984375,
    -5.21875,
    -7.28125,
    -4.05859375,
    -5.59375,
    -2.583984375,
    -5.7109375,
    -4.72265625,
    -7.21484375,
    -4.75390625,
    -5.1484375,
    -6.4921875,
    -5.2578125,
    -3.154296875,
    -5.53125,
    -5.37890625,
    -5.98046875,
    -4.22265625,
    -3.12890625,
    -7.390625,
    -4.453125,
    -4.79296875,
    -6.2421875,
    -4.91015625,
    -5.328125,
    -4.64453125,
    -4.58203125,
    -5.2890625,
    -4.91796875,
    -5.96875,
    -4.9140625,
    -2.97265625,
    -5.17578125,
    -2.921875,
    -3.38671875,
    -3.6796875,
    -4.61328125,
    -4.91015625,
    -5.8828125,
    -7.8515625,
    -6.65625,
    -7.796875,
    -6.88671875,
    -3.783203125,
    -7.609375,
    -7.79296875,
    -2.384765625,
    -8.0703125,
    -4.2578125,
    -8.140625,
    -5.6171875,
    -2.7890625,
    -6.42578125,
    -3.818359375,
    -6.55859375,
    -4.3671875,
    -7.0390625,
    -4.41796875,
    -5.890625,
    -4.41015625,
    -4.3671875,
    -2.841796875,
    -4.3828125,
    -3.8125,
    -3.98828125,
    -2.625,
    -3.96875,
    -2.92578125,
    -5.1953125,
    -3.712890625,
    -4.35546875,
    -4.50390625,
    -4.3671875,
    -4.76953125,
    -4.97265625,
    -5.45703125,
    -4.0078125,
    -4.36328125,
    -4.3671875,
    -6.48828125,
    -4.66796875,
    -3.921875,
    -4.4140625,
    -4.6875,
    -6.50390625,
    -4.6484375,
    -5.984375,
    -4.48046875,
    -5.78515625,
    -4.421875,
    -4.87109375,
    -5.5625,
    -7.00390625,
    -4.97265625,
    -4.375,
    -4.85546875,
    -6.37109375,
    -4.4296875,
    -3.873046875,
    -3.595703125,
    -4.01953125,
    -3.3671875,
    -3.939453125,
    -2.546875,
    -6.14453125,
    -6.09375,
    -3.484375,
    -3.33203125,
    -5.8125,
    -6.0,
    -4.05078125,
    -5.78125,
    -6.05859375,
    -5.19921875,
    -5.05859375,
    -4.28125,
    -5.39453125,
    -6.03125,
    -3.31640625,
    -3.826171875,
    -4.23046875,
    -4.08984375,
    -3.30859375,
    -5.1171875,
    -5.66015625,
    -6.390625,
    -5.33984375,
    -4.78125,
    -4.7265625,
    -4.94140625,
    -3.390625,
    -1.2822265625,
    -3.640625,
    -2.72265625,
    -1.72265625,
    -2.234375,
    -2.12890625,
    -3.57421875,
    -0.47412109375,
    -2.1484375,
    -2.09375,
    -1.7900390625,
    -1.4033203125,
    -2.591796875,
    -2.482421875,
    -3.06640625,
    -1.87890625,
    -3.541015625,
    -2.060546875,
    -2.439453125,
    -2.6328125,
    -2.623046875,
    -3.078125,
    -0.9765625,
    -2.041015625,
    -3.390625,
    -3.515625,
    -3.32421875,
    -3.806640625,
    -3.48828125,
    -3.52734375,
    -2.830078125,
    -2.18359375,
    -2.677734375,
    -1.748046875,
    -1.3115234375,
    -2.001953125,
    -2.01171875,
    -2.34375,
    -1.119140625,
    -1.87890625,
    -3.18359375,
    -1.9814453125,
    -1.6455078125,
    -2.111328125,
    -2.158203125,
    -2.18359375,
    -1.689453125,
    -1.5400390625,
    -1.2724609375,
    -2.0703125,
    -1.9296875,
    -2.0234375,
    -2.697265625,
    -1.177734375,
    -3.044921875,
    -2.66015625,
    -0.467529296875,
    -2.607421875,
    -1.1572265625,
    -1.306640625,
    -0.9814453125,
    -1.8564453125,
    -2.46484375,
    -1.474609375,
    -1.8525390625,
    -2.501953125,
    -1.9970703125,
    -1.9619140625,
    -1.2333984375,
    -1.830078125,
    -1.41015625,
    -1.5732421875,
    -0.322021484375,
    0.1839599609375,
    -0.4541015625,
    0.96240234375,
    1.2138671875,
    -0.00055694580078125,
    -1.060546875,
    -0.59033203125,
    0.2445068359375,
    -0.74755859375,
    0.6142578125,
    1.1865234375,
    1.404296875,
    -0.76708984375,
    -0.01213836669921875,
    -0.89111328125,
    0.326171875,
    0.7373046875,
    -0.8935546875,
    -0.2398681640625,
    0.158935546875,
    0.7353515625,
    0.35400390625,
    -0.276611328125,
    -0.99951171875,
    1.4248046875,
    0.096435546875,
    -0.85986328125,
    -0.154296875,
    0.41552734375,
    -0.494384765625,
    -3.935546875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0012984982458874583,
    0.995904266834259,
    0.0016233117785304785,
    0.0007774500991217792,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003964500210713595
   ],
   "p": [
    0.0012990132672712207,
    0.9962992668151855,
    0.0016239556716755033,
    0.0007777584833092988
   ],
   "p_oracle": [
    0.0012890654616057873,
    0.9963335990905762,
    0.0016073038568720222,
    0.0007700907881371677
   ],
   "abstain": 0.0003964500210713595,
   "abstain_oracle": 0.0003932055551558733,
   "abs_delta_abstain": 3.2444659154862165e-06,
   "oracle_margin": 0.994726300239563,
   "max_abs_delta_p": 3.4332275390625e-05,
   "mean_abs_delta_p": 1.7149897757917643e-05,
   "max_abs_delta_raw_logits": 0.00846719741821289,
   "mean_abs_delta_raw_logits": 0.00235658569727093,
   "logits_min": -8.140625,
   "logits_max": 4.3515625,
   "raw_logits_min": -8.140625,
   "raw_logits_max": 4.3515625,
   "raw_logits_absmax": 8.140625,
   "oracle_logits_min": -8.147294044494629,
   "oracle_logits_max": 4.35848331451416,
   "oracle_raw_logits_absmax": 8.147294044494629,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.25785549997817725,
   "seconds": 0.25785549997817725,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r10-white",
   "request_id": "r10",
   "question_id": "white",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 38,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -3.013671875,
    0.82568359375,
    -1.037109375,
    -1.396484375,
    -1.103515625,
    -2.447265625,
    -1.845703125,
    -2.6328125,
    -3.00390625,
    -1.5380859375,
    -2.263671875,
    -3.42578125,
    -4.18359375,
    -2.423828125,
    -3.0,
    -4.28515625,
    -3.9609375,
    -4.65625,
    -2.595703125,
    -2.41015625,
    -3.29296875,
    -1.583984375,
    -4.83203125,
    -6.12890625,
    -3.568359375,
    -4.44921875,
    -2.431640625,
    -4.75390625,
    -3.68359375,
    -5.3515625,
    -4.1796875,
    -3.7265625,
    -5.578125,
    -4.14453125,
    -2.537109375,
    -4.546875,
    -4.97265625,
    -5.61328125,
    -4.1171875,
    -2.703125,
    -6.6015625,
    -3.9375,
    -4.484375,
    -5.7578125,
    -3.53125,
    -4.38671875,
    -4.1328125,
    -4.35546875,
    -4.61328125,
    -4.1640625,
    -5.0703125,
    -4.18359375,
    -2.5390625,
    -4.828125,
    -2.400390625,
    -3.416015625,
    -3.3359375,
    -4.40625,
    -4.1015625,
    -5.16015625,
    -6.81640625,
    -6.04296875,
    -7.23828125,
    -5.8984375,
    -3.107421875,
    -7.0546875,
    -6.99609375,
    -2.193359375,
    -7.3046875,
    -4.13671875,
    -7.24609375,
    -4.96484375,
    -2.2265625,
    -5.3828125,
    -3.578125,
    -5.55859375,
    -4.08984375,
    -6.12109375,
    -3.73046875,
    -5.35546875,
    -4.15234375,
    -3.6328125,
    -2.46875,
    -3.953125,
    -3.03125,
    -3.857421875,
    -2.3828125,
    -3.255859375,
    -2.759765625,
    -4.1015625,
    -3.154296875,
    -3.724609375,
    -3.302734375,
    -3.875,
    -4.3515625,
    -4.62109375,
    -4.5078125,
    -3.365234375,
    -3.146484375,
    -4.01953125,
    -5.16015625,
    -4.484375,
    -3.119140625,
    -3.373046875,
    -4.125,
    -5.36328125,
    -3.197265625,
    -4.90234375,
    -3.828125,
    -4.95703125,
    -4.0625,
    -4.21875,
    -4.33984375,
    -5.75390625,
    -4.19140625,
    -3.45703125,
    -4.4765625,
    -5.2109375,
    -3.94140625,
    -2.9375,
    -3.041015625,
    -3.98046875,
    -2.96484375,
    -3.373046875,
    -2.275390625,
    -5.16796875,
    -4.87109375,
    -2.7109375,
    -2.802734375,
    -4.796875,
    -5.20703125,
    -2.779296875,
    -5.36328125,
    -5.64453125,
    -4.8671875,
    -4.125,
    -3.412109375,
    -4.32421875,
    -5.421875,
    -2.255859375,
    -3.140625,
    -3.931640625,
    -3.3125,
    -3.197265625,
    -4.15625,
    -4.67578125,
    -5.30859375,
    -4.7890625,
    -4.84375,
    -4.109375,
    -4.27734375,
    -2.693359375,
    -0.775390625,
    -2.830078125,
    -1.998046875,
    -1.017578125,
    -1.4482421875,
    -1.783203125,
    -3.01171875,
    -0.468017578125,
    -1.8505859375,
    -1.93359375,
    -1.1826171875,
    -1.2939453125,
    -2.703125,
    -2.12109375,
    -2.263671875,
    -1.4833984375,
    -3.2421875,
    -1.9580078125,
    -1.5029296875,
    -2.25390625,
    -1.7431640625,
    -2.671875,
    -0.57666015625,
    -1.6650390625,
    -2.884765625,
    -3.05078125,
    -2.671875,
    -2.759765625,
    -2.806640625,
    -2.40234375,
    -2.4453125,
    -1.81640625,
    -2.1328125,
    -1.357421875,
    -1.2861328125,
    -1.330078125,
    -1.451171875,
    -1.7607421875,
    -0.498046875,
    -1.5693359375,
    -2.322265625,
    -1.2109375,
    -1.3427734375,
    -1.8125,
    -1.7734375,
    -1.22265625,
    -1.328125,
    -0.97607421875,
    -1.1455078125,
    -1.478515625,
    -1.3798828125,
    -1.4423828125,
    -1.9462890625,
    -0.53662109375,
    -2.40234375,
    -2.220703125,
    -0.085693359375,
    -1.642578125,
    -0.4873046875,
    -0.92236328125,
    -0.336181640625,
    -0.865234375,
    -2.337890625,
    -0.86328125,
    -1.4248046875,
    -2.115234375,
    -1.423828125,
    -1.412109375,
    -1.1884765625,
    -1.5859375,
    -1.3701171875,
    -1.1455078125,
    -0.0723876953125,
    0.384521484375,
    -0.32421875,
    1.0,
    1.0302734375,
    -0.09527587890625,
    -0.650390625,
    -0.37158203125,
    -0.2332763671875,
    -0.87744140625,
    0.475830078125,
    1.27734375,
    1.6015625,
    -0.578125,
    -0.425537109375,
    -0.72802734375,
    0.68115234375,
    0.63720703125,
    -0.693359375,
    -0.01410675048828125,
    -0.0328369140625,
    0.41650390625,
    0.346435546875,
    -0.66357421875,
    -0.8408203125,
    1.5615234375,
    -0.1065673828125,
    -0.8525390625,
    -0.004283905029296875,
    0.53857421875,
    -0.56787109375,
    -7.421875
   ],
   "raw256_argmax": 236,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.021587323397397995,
    0.9781419634819031,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0002707751118578017
   ],
   "p": [
    0.021593168377876282,
    0.9784068465232849
   ],
   "p_oracle": [
    0.021688533946871758,
    0.9783114194869995
   ],
   "abstain": 0.0002707751118578017,
   "abstain_oracle": 0.00026848967536352575,
   "abs_delta_abstain": 2.2854364942759275e-06,
   "oracle_margin": 0.9566228985786438,
   "max_abs_delta_p": 9.542703628540039e-05,
   "mean_abs_delta_p": 9.539630264043808e-05,
   "max_abs_delta_raw_logits": 0.011828422546386719,
   "mean_abs_delta_raw_logits": 0.0011631173256319016,
   "logits_min": -7.421875,
   "logits_max": 1.6015625,
   "raw_logits_min": -7.421875,
   "raw_logits_max": 1.6015625,
   "raw_logits_absmax": 7.421875,
   "oracle_logits_min": -7.433703422546387,
   "oracle_logits_max": 1.6013541221618652,
   "oracle_raw_logits_absmax": 7.433703422546387,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.21958858298603445,
   "seconds": 0.21958858298603445,
   "full_vocab_argmax_id": 236,
   "full_vocab_argmax_is_label": false
  },
  {
   "id": "r10-damage",
   "request_id": "r10",
   "question_id": "damage",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 67,
   "nopts": 5,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    4.83203125,
    -1.208984375,
    -2.171875,
    -3.47265625,
    -3.1875,
    -2.90625,
    -3.28515625,
    -4.203125,
    -3.275390625,
    -2.66015625,
    -2.8125,
    -3.62109375,
    -4.8125,
    -3.02734375,
    -4.296875,
    -4.52734375,
    -4.21875,
    -4.5078125,
    -2.966796875,
    -3.59765625,
    -3.58203125,
    -1.4970703125,
    -5.09765625,
    -6.2734375,
    -3.904296875,
    -4.3671875,
    -2.974609375,
    -5.2734375,
    -4.17578125,
    -6.703125,
    -4.1875,
    -5.70703125,
    -5.75,
    -4.66796875,
    -3.248046875,
    -4.91796875,
    -5.0625,
    -6.30078125,
    -4.21875,
    -3.53515625,
    -6.58203125,
    -4.359375,
    -5.33203125,
    -6.13671875,
    -4.328125,
    -5.05859375,
    -4.0546875,
    -4.8125,
    -5.38671875,
    -4.42578125,
    -5.6640625,
    -4.32421875,
    -3.421875,
    -5.78515625,
    -3.248046875,
    -3.4453125,
    -4.36328125,
    -4.8984375,
    -4.90234375,
    -6.79296875,
    -7.69140625,
    -7.20703125,
    -8.3515625,
    -5.81640625,
    -3.875,
    -7.359375,
    -8.0859375,
    -2.3828125,
    -7.65234375,
    -4.078125,
    -8.3203125,
    -5.54296875,
    -3.615234375,
    -5.70703125,
    -3.701171875,
    -6.55859375,
    -4.359375,
    -7.1171875,
    -4.04296875,
    -6.1171875,
    -4.85546875,
    -4.21484375,
    -3.185546875,
    -4.859375,
    -3.6328125,
    -4.48828125,
    -2.46484375,
    -4.234375,
    -3.177734375,
    -5.10546875,
    -3.751953125,
    -4.45703125,
    -4.328125,
    -3.822265625,
    -4.58203125,
    -4.46484375,
    -5.0859375,
    -3.744140625,
    -3.296875,
    -4.33984375,
    -5.921875,
    -4.9609375,
    -3.466796875,
    -4.32421875,
    -4.5703125,
    -6.2109375,
    -4.25390625,
    -5.6171875,
    -4.52734375,
    -6.25,
    -4.82421875,
    -4.84375,
    -5.37109375,
    -6.46484375,
    -5.1875,
    -4.1171875,
    -4.83984375,
    -5.86328125,
    -4.0390625,
    -4.26953125,
    -3.4921875,
    -4.5078125,
    -3.16796875,
    -3.75390625,
    -2.529296875,
    -5.52734375,
    -5.9140625,
    -3.162109375,
    -3.08203125,
    -6.06640625,
    -6.359375,
    -2.9609375,
    -5.78515625,
    -6.53515625,
    -5.08984375,
    -4.171875,
    -3.970703125,
    -5.19921875,
    -6.375,
    -3.3671875,
    -3.666015625,
    -4.42578125,
    -3.6484375,
    -2.2734375,
    -5.4140625,
    -5.546875,
    -6.1640625,
    -5.72265625,
    -5.1640625,
    -4.95703125,
    -4.7109375,
    -2.830078125,
    -1.2626953125,
    -3.83984375,
    -3.015625,
    -1.654296875,
    -1.892578125,
    -1.966796875,
    -2.953125,
    -0.54052734375,
    -1.765625,
    -2.23828125,
    -1.9111328125,
    -1.0107421875,
    -3.03515625,
    -1.658203125,
    -2.947265625,
    -2.708984375,
    -3.892578125,
    -1.9130859375,
    -2.642578125,
    -2.705078125,
    -2.384765625,
    -3.11328125,
    -0.95751953125,
    -2.375,
    -4.12890625,
    -3.45703125,
    -2.97265625,
    -2.572265625,
    -3.361328125,
    -2.96875,
    -3.185546875,
    -2.29296875,
    -2.517578125,
    -1.91015625,
    -1.251953125,
    -1.630859375,
    -2.34765625,
    -3.03515625,
    -0.81689453125,
    -1.8447265625,
    -2.80078125,
    -1.796875,
    -1.9501953125,
    -1.8037109375,
    -2.279296875,
    -2.109375,
    -2.142578125,
    -1.3193359375,
    -1.232421875,
    -2.078125,
    -1.7158203125,
    -2.041015625,
    -2.52734375,
    -1.2421875,
    -2.4765625,
    -2.619140625,
    -0.0175628662109375,
    -2.4140625,
    -1.8388671875,
    -1.294921875,
    -0.005062103271484375,
    -0.861328125,
    -2.767578125,
    -1.0947265625,
    -1.78515625,
    -2.013671875,
    -1.2607421875,
    -2.009765625,
    -0.98876953125,
    -1.6357421875,
    -0.64453125,
    -1.18359375,
    -0.323486328125,
    -0.294677734375,
    -0.75048828125,
    1.0771484375,
    0.87353515625,
    -0.382568359375,
    -0.326171875,
    -0.666015625,
    0.81494140625,
    -0.71337890625,
    0.3740234375,
    0.51708984375,
    1.2080078125,
    -0.26953125,
    0.4599609375,
    -0.8798828125,
    0.1444091796875,
    0.8037109375,
    -1.0732421875,
    -0.57373046875,
    -0.1097412109375,
    0.91943359375,
    0.486328125,
    -0.4208984375,
    -0.7265625,
    2.40625,
    -0.0300140380859375,
    -0.024688720703125,
    -0.240478515625,
    0.199462890625,
    -0.50927734375,
    -4.01953125
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9947463274002075,
    0.003038334660232067,
    0.0012071243254467845,
    0.00034689123276621103,
    0.00045594602124765515,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00020535766088869423
   ],
   "p": [
    0.9949506521224976,
    0.0030389586463570595,
    0.0012073722900822759,
    0.0003469624789431691,
    0.0004560396773740649
   ],
   "p_oracle": [
    0.9949960112571716,
    0.0030125249177217484,
    0.0011956130620092154,
    0.00034420800511725247,
    0.0004516497428994626
   ],
   "abstain": 0.00020535766088869423,
   "abstain_oracle": 0.00020448834402486682,
   "abs_delta_abstain": 8.693168638274074e-07,
   "oracle_margin": 0.9919834733009338,
   "max_abs_delta_p": 4.5359134674072266e-05,
   "mean_abs_delta_p": 1.8139299936592578e-05,
   "max_abs_delta_raw_logits": 0.008416175842285156,
   "mean_abs_delta_raw_logits": 0.0022654111380688846,
   "logits_min": -8.3515625,
   "logits_max": 4.83203125,
   "raw_logits_min": -8.3515625,
   "raw_logits_max": 4.83203125,
   "raw_logits_absmax": 8.3515625,
   "oracle_logits_min": -8.355578422546387,
   "oracle_logits_max": 4.839208602905273,
   "oracle_raw_logits_absmax": 8.355578422546387,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3811905410839245,
   "seconds": 0.3811905410839245,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r11-status",
   "request_id": "r11",
   "question_id": "status",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 37,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.80859375,
    3.21484375,
    -2.43359375,
    -2.560546875,
    -2.181640625,
    -2.837890625,
    -2.73828125,
    -3.576171875,
    -3.716796875,
    -2.44921875,
    -2.994140625,
    -3.966796875,
    -5.328125,
    -3.14453125,
    -4.03125,
    -4.3515625,
    -3.69140625,
    -4.31640625,
    -2.91796875,
    -3.05859375,
    -3.412109375,
    -1.9912109375,
    -5.26953125,
    -6.859375,
    -3.91015625,
    -5.3671875,
    -2.41796875,
    -5.19140625,
    -4.4609375,
    -6.58203125,
    -4.45703125,
    -4.38671875,
    -6.1875,
    -4.89453125,
    -3.060546875,
    -5.35546875,
    -5.24609375,
    -5.87890625,
    -4.203125,
    -2.90625,
    -7.19921875,
    -4.29296875,
    -4.66015625,
    -6.0078125,
    -4.2578125,
    -5.046875,
    -4.5,
    -4.49609375,
    -5.36328125,
    -4.65625,
    -5.67578125,
    -4.67578125,
    -2.734375,
    -4.87109375,
    -2.66015625,
    -3.3359375,
    -3.52734375,
    -4.39453125,
    -4.70703125,
    -5.421875,
    -7.55078125,
    -6.3515625,
    -7.62890625,
    -6.51953125,
    -3.609375,
    -7.4375,
    -7.67578125,
    -2.3125,
    -7.74609375,
    -4.07421875,
    -7.7890625,
    -5.28125,
    -2.470703125,
    -6.16015625,
    -3.724609375,
    -6.17578125,
    -4.2109375,
    -6.68359375,
    -4.2421875,
    -5.65625,
    -4.23828125,
    -4.11328125,
    -2.673828125,
    -4.2265625,
    -3.513671875,
    -3.90234375,
    -2.58984375,
    -3.802734375,
    -2.998046875,
    -4.84375,
    -3.4375,
    -4.07421875,
    -4.203125,
    -4.20703125,
    -4.58203125,
    -4.6484375,
    -5.12890625,
    -3.751953125,
    -3.97265625,
    -4.13671875,
    -6.046875,
    -4.5,
    -3.73046875,
    -4.2109375,
    -4.47265625,
    -5.984375,
    -4.19921875,
    -5.59765625,
    -4.12890625,
    -5.5703125,
    -4.32421875,
    -4.58984375,
    -4.94140625,
    -6.453125,
    -4.74609375,
    -4.00390625,
    -4.5703125,
    -6.01953125,
    -4.4765625,
    -3.48828125,
    -3.373046875,
    -3.96875,
    -3.27734375,
    -3.78125,
    -2.365234375,
    -5.8203125,
    -5.7265625,
    -3.103515625,
    -3.095703125,
    -5.3046875,
    -5.55078125,
    -3.66796875,
    -5.62890625,
    -5.94921875,
    -4.91796875,
    -4.67578125,
    -3.921875,
    -4.94140625,
    -5.80078125,
    -2.88671875,
    -3.712890625,
    -4.13671875,
    -3.693359375,
    -3.330078125,
    -4.671875,
    -5.1875,
    -6.0859375,
    -5.1484375,
    -4.62109375,
    -4.37890625,
    -4.7421875,
    -3.1328125,
    -1.12890625,
    -3.41796875,
    -2.443359375,
    -1.58984375,
    -2.005859375,
    -1.9189453125,
    -3.41015625,
    -0.43701171875,
    -2.123046875,
    -1.9033203125,
    -1.66015625,
    -1.265625,
    -2.560546875,
    -2.30078125,
    -2.857421875,
    -1.7666015625,
    -3.42578125,
    -2.029296875,
    -2.298828125,
    -2.529296875,
    -2.4296875,
    -2.93359375,
    -0.84716796875,
    -1.9375,
    -3.193359375,
    -3.275390625,
    -3.12109375,
    -3.51171875,
    -3.388671875,
    -3.1640625,
    -2.6875,
    -2.189453125,
    -2.509765625,
    -1.6328125,
    -1.326171875,
    -1.755859375,
    -1.92578125,
    -2.087890625,
    -0.94677734375,
    -1.755859375,
    -2.861328125,
    -1.697265625,
    -1.3798828125,
    -2.15625,
    -2.04296875,
    -1.8681640625,
    -1.587890625,
    -1.4091796875,
    -1.3427734375,
    -1.943359375,
    -1.7138671875,
    -1.8994140625,
    -2.259765625,
    -0.97216796875,
    -2.62109375,
    -2.58984375,
    -0.38525390625,
    -2.181640625,
    -0.92431640625,
    -1.26953125,
    -0.771484375,
    -1.3740234375,
    -2.52734375,
    -1.2041015625,
    -1.7509765625,
    -2.328125,
    -1.8232421875,
    -1.912109375,
    -1.5244140625,
    -1.7099609375,
    -1.4599609375,
    -1.3232421875,
    -0.2054443359375,
    0.10260009765625,
    -0.4775390625,
    0.82421875,
    1.078125,
    -0.035430908203125,
    -0.8447265625,
    -0.478515625,
    0.02947998046875,
    -0.693359375,
    0.552734375,
    1.26953125,
    1.470703125,
    -0.71826171875,
    -0.0972900390625,
    -0.8984375,
    0.3876953125,
    0.8232421875,
    -0.83349609375,
    -0.177490234375,
    0.052093505859375,
    0.45751953125,
    0.316162109375,
    -0.45654296875,
    -0.95263671875,
    1.5244140625,
    -0.0187225341796875,
    -0.81201171875,
    -0.169677734375,
    0.419921875,
    -0.51611328125,
    -4.2109375
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0033479430712759495,
    0.9909907579421997,
    0.0047712549567222595,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0008900723769329488
   ],
   "p": [
    0.0033509256318211555,
    0.9918736219406128,
    0.0047755055129528046
   ],
   "p_oracle": [
    0.0033252013381570578,
    0.9919410347938538,
    0.004733750596642494
   ],
   "abstain": 0.0008900723769329488,
   "abstain_oracle": 0.0008843487594276667,
   "abs_delta_abstain": 5.723617505282164e-06,
   "oracle_margin": 0.987207293510437,
   "max_abs_delta_p": 6.74128532409668e-05,
   "mean_abs_delta_p": 4.496402107179165e-05,
   "max_abs_delta_raw_logits": 0.00696563720703125,
   "mean_abs_delta_raw_logits": 0.0018397607491351664,
   "logits_min": -7.7890625,
   "logits_max": 3.21484375,
   "raw_logits_min": -7.7890625,
   "raw_logits_max": 3.21484375,
   "raw_logits_absmax": 7.7890625,
   "oracle_logits_min": -7.790453910827637,
   "oracle_logits_max": 3.2206735610961914,
   "oracle_raw_logits_absmax": 7.790453910827637,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3990803330671042,
   "seconds": 0.3990803330671042,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r11-restart",
   "request_id": "r11",
   "question_id": "restart",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 42,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    3.21484375,
    -1.8955078125,
    -1.8984375,
    -2.75390625,
    -2.123046875,
    -2.55859375,
    -2.681640625,
    -3.740234375,
    -2.984375,
    -2.2265625,
    -2.486328125,
    -3.21875,
    -4.1875,
    -2.548828125,
    -3.681640625,
    -4.22265625,
    -4.11328125,
    -4.64453125,
    -2.5,
    -3.052734375,
    -3.11328125,
    -1.3447265625,
    -4.77734375,
    -5.91015625,
    -3.41015625,
    -4.1640625,
    -2.716796875,
    -4.86328125,
    -3.86328125,
    -6.359375,
    -3.9375,
    -4.7890625,
    -5.89453125,
    -4.26171875,
    -2.783203125,
    -4.703125,
    -4.953125,
    -6.07421875,
    -4.24609375,
    -3.32421875,
    -6.3515625,
    -4.09765625,
    -5.16015625,
    -5.90625,
    -4.0625,
    -4.6796875,
    -3.982421875,
    -4.7734375,
    -5.1484375,
    -3.962890625,
    -5.25390625,
    -4.16796875,
    -3.0390625,
    -5.31640625,
    -3.07421875,
    -3.349609375,
    -4.015625,
    -4.6875,
    -4.6875,
    -6.43359375,
    -7.1796875,
    -6.9375,
    -7.890625,
    -5.671875,
    -3.71484375,
    -7.03125,
    -7.390625,
    -2.43359375,
    -7.26953125,
    -4.046875,
    -7.9609375,
    -5.0625,
    -3.2109375,
    -5.3515625,
    -3.525390625,
    -6.140625,
    -4.01953125,
    -6.91796875,
    -3.69140625,
    -5.72265625,
    -4.59765625,
    -3.8359375,
    -2.87890625,
    -4.4140625,
    -3.42578125,
    -4.33203125,
    -2.359375,
    -3.9609375,
    -3.046875,
    -4.49609375,
    -3.505859375,
    -4.265625,
    -4.11328125,
    -3.396484375,
    -4.2890625,
    -4.40625,
    -4.796875,
    -3.30078125,
    -2.998046875,
    -4.15625,
    -5.3515625,
    -4.75,
    -3.234375,
    -3.892578125,
    -4.53125,
    -5.66015625,
    -3.71484375,
    -5.07421875,
    -4.19140625,
    -5.8046875,
    -4.484375,
    -4.61328125,
    -4.93359375,
    -5.984375,
    -4.77734375,
    -3.966796875,
    -4.6796875,
    -5.59765625,
    -3.7109375,
    -3.755859375,
    -3.259765625,
    -4.3359375,
    -3.033203125,
    -3.7890625,
    -2.265625,
    -5.203125,
    -5.640625,
    -2.712890625,
    -2.802734375,
    -5.4375,
    -5.953125,
    -2.693359375,
    -5.609375,
    -6.28125,
    -4.984375,
    -3.833984375,
    -3.548828125,
    -4.703125,
    -5.82421875,
    -2.7734375,
    -3.267578125,
    -4.16796875,
    -3.291015625,
    -2.466796875,
    -4.88671875,
    -5.0078125,
    -5.66015625,
    -5.36328125,
    -4.9921875,
    -4.52734375,
    -4.41015625,
    -2.69140625,
    -1.0205078125,
    -3.56640625,
    -2.9921875,
    -1.396484375,
    -1.765625,
    -1.8583984375,
    -2.91015625,
    -0.430419921875,
    -1.755859375,
    -2.125,
    -1.6943359375,
    -0.88037109375,
    -2.8125,
    -1.751953125,
    -2.6640625,
    -2.44140625,
    -3.74609375,
    -1.8828125,
    -2.224609375,
    -2.48046875,
    -2.13671875,
    -3.01953125,
    -0.5546875,
    -1.9521484375,
    -3.650390625,
    -3.19140625,
    -2.81640625,
    -2.228515625,
    -3.197265625,
    -2.275390625,
    -2.822265625,
    -1.947265625,
    -2.33984375,
    -1.65625,
    -1.3818359375,
    -1.453125,
    -2.154296875,
    -2.5703125,
    -0.483642578125,
    -1.6728515625,
    -2.37890625,
    -1.51953125,
    -1.57421875,
    -1.724609375,
    -2.125,
    -1.8330078125,
    -1.982421875,
    -1.044921875,
    -1.193359375,
    -1.7353515625,
    -1.6650390625,
    -1.763671875,
    -2.25390625,
    -0.88525390625,
    -2.0859375,
    -2.380859375,
    0.10687255859375,
    -1.8466796875,
    -1.263671875,
    -1.0029296875,
    0.058929443359375,
    -0.5419921875,
    -2.58203125,
    -0.947265625,
    -1.5830078125,
    -1.84765625,
    -1.0751953125,
    -1.7578125,
    -0.82958984375,
    -1.70703125,
    -0.6865234375,
    -1.087890625,
    -0.2286376953125,
    -0.144775390625,
    -0.56884765625,
    1.0634765625,
    0.7734375,
    -0.29833984375,
    -0.41455078125,
    -0.7197265625,
    0.4052734375,
    -0.74853515625,
    0.33154296875,
    0.52978515625,
    1.322265625,
    -0.422119140625,
    0.1221923828125,
    -0.7626953125,
    0.443359375,
    0.814453125,
    -0.94775390625,
    -0.4755859375,
    -0.131103515625,
    0.830078125,
    0.57470703125,
    -0.77001953125,
    -0.615234375,
    2.27734375,
    -0.256103515625,
    -0.30615234375,
    -0.393798828125,
    0.252685546875,
    -0.40478515625,
    -4.84765625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9934654831886292,
    0.006204021163284779,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003305136924609542
   ],
   "p": [
    0.9937939643859863,
    0.006206072401255369
   ],
   "p_oracle": [
    0.9938108921051025,
    0.006189099978655577
   ],
   "abstain": 0.0003305136924609542,
   "abstain_oracle": 0.00033229435211978853,
   "abs_delta_abstain": 1.7806596588343382e-06,
   "oracle_margin": 0.9876217842102051,
   "max_abs_delta_p": 1.697242259979248e-05,
   "mean_abs_delta_p": 1.695007085800171e-05,
   "max_abs_delta_raw_logits": 0.007889747619628906,
   "mean_abs_delta_raw_logits": 0.00280510209267959,
   "logits_min": -7.9609375,
   "logits_max": 3.21484375,
   "raw_logits_min": -7.9609375,
   "raw_logits_max": 3.21484375,
   "raw_logits_absmax": 7.9609375,
   "oracle_logits_min": -7.964931488037109,
   "oracle_logits_max": 3.2155966758728027,
   "oracle_raw_logits_absmax": 7.964931488037109,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2357998340157792,
   "seconds": 0.2357998340157792,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r11-urgency",
   "request_id": "r11",
   "question_id": "urgency",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 46,
   "nopts": 3,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    4.09765625,
    -0.87158203125,
    -2.369140625,
    -2.716796875,
    -2.35546875,
    -2.986328125,
    -2.580078125,
    -3.669921875,
    -3.251953125,
    -2.083984375,
    -2.701171875,
    -3.419921875,
    -4.359375,
    -2.7421875,
    -3.796875,
    -4.03515625,
    -3.8515625,
    -4.55078125,
    -2.861328125,
    -3.251953125,
    -3.01171875,
    -1.5712890625,
    -4.99609375,
    -6.0,
    -3.34375,
    -4.19140625,
    -2.498046875,
    -4.6796875,
    -4.015625,
    -6.3984375,
    -4.04296875,
    -4.765625,
    -5.8671875,
    -4.37890625,
    -2.955078125,
    -4.5859375,
    -5.15625,
    -6.1484375,
    -4.2109375,
    -3.212890625,
    -6.57421875,
    -4.4609375,
    -5.05078125,
    -5.91796875,
    -4.0078125,
    -4.56640625,
    -3.916015625,
    -4.69140625,
    -5.3671875,
    -4.28125,
    -5.42578125,
    -4.1640625,
    -3.11328125,
    -5.46875,
    -3.09765625,
    -3.4140625,
    -3.87890625,
    -4.68359375,
    -4.5390625,
    -6.33203125,
    -7.26171875,
    -6.72265625,
    -7.87109375,
    -5.67578125,
    -3.74609375,
    -7.11328125,
    -7.45703125,
    -2.310546875,
    -7.234375,
    -4.07421875,
    -7.83984375,
    -5.12890625,
    -3.056640625,
    -5.4140625,
    -3.68359375,
    -5.9921875,
    -4.07421875,
    -6.7890625,
    -3.6328125,
    -5.75,
    -4.51171875,
    -3.900390625,
    -2.853515625,
    -4.53515625,
    -3.298828125,
    -4.3515625,
    -2.21484375,
    -4.14453125,
    -2.98828125,
    -4.609375,
    -3.6015625,
    -4.28125,
    -4.09765625,
    -3.560546875,
    -4.29296875,
    -4.296875,
    -4.9453125,
    -3.400390625,
    -2.9765625,
    -4.14453125,
    -5.4765625,
    -4.640625,
    -3.1796875,
    -3.818359375,
    -4.41015625,
    -5.6640625,
    -3.7109375,
    -5.28125,
    -4.05859375,
    -6.00390625,
    -4.5234375,
    -4.5703125,
    -4.9140625,
    -6.125,
    -4.81640625,
    -3.9453125,
    -4.6328125,
    -5.65625,
    -3.794921875,
    -3.611328125,
    -3.154296875,
    -4.296875,
    -3.154296875,
    -3.75390625,
    -2.384765625,
    -5.15234375,
    -5.59375,
    -2.66015625,
    -2.953125,
    -5.46875,
    -5.875,
    -2.703125,
    -5.546875,
    -6.1875,
    -4.7890625,
    -3.98828125,
    -3.55078125,
    -4.828125,
    -5.90625,
    -2.921875,
    -3.4453125,
    -4.203125,
    -3.20703125,
    -2.515625,
    -4.9609375,
    -5.0859375,
    -5.6953125,
    -5.26171875,
    -5.0546875,
    -4.56640625,
    -4.30859375,
    -2.798828125,
    -1.0751953125,
    -3.513671875,
    -2.974609375,
    -1.5634765625,
    -1.77734375,
    -1.9228515625,
    -2.98828125,
    -0.59326171875,
    -1.701171875,
    -2.474609375,
    -1.7568359375,
    -1.03515625,
    -2.775390625,
    -1.8095703125,
    -2.6640625,
    -2.423828125,
    -3.666015625,
    -1.80078125,
    -2.158203125,
    -2.5546875,
    -2.255859375,
    -2.974609375,
    -0.63525390625,
    -2.052734375,
    -3.775390625,
    -3.376953125,
    -2.833984375,
    -2.369140625,
    -3.228515625,
    -2.521484375,
    -2.73828125,
    -2.328125,
    -2.490234375,
    -1.9443359375,
    -1.234375,
    -1.5419921875,
    -2.3828125,
    -2.72265625,
    -0.5380859375,
    -1.7177734375,
    -2.51953125,
    -1.6708984375,
    -1.6953125,
    -1.5537109375,
    -2.056640625,
    -1.8056640625,
    -2.134765625,
    -0.94921875,
    -1.0859375,
    -1.8056640625,
    -1.4287109375,
    -1.96875,
    -2.3828125,
    -1.0810546875,
    -2.35546875,
    -2.53515625,
    0.001819610595703125,
    -2.123046875,
    -1.341796875,
    -1.185546875,
    -0.00742340087890625,
    -0.5771484375,
    -2.6640625,
    -1.013671875,
    -1.7373046875,
    -1.8466796875,
    -1.28125,
    -1.96875,
    -1.041015625,
    -1.7294921875,
    -0.82275390625,
    -1.1396484375,
    -0.2734375,
    -0.1988525390625,
    -0.6982421875,
    1.3623046875,
    0.84912109375,
    -0.306884765625,
    -0.4052734375,
    -0.61328125,
    0.65673828125,
    -0.728515625,
    0.296142578125,
    0.6474609375,
    1.365234375,
    -0.2274169921875,
    0.33154296875,
    -0.9267578125,
    0.31005859375,
    0.5888671875,
    -0.814453125,
    -0.22021484375,
    0.0130157470703125,
    0.65966796875,
    0.60205078125,
    -0.434326171875,
    -0.763671875,
    2.2734375,
    -0.228515625,
    -0.297119140625,
    -0.2861328125,
    0.272705078125,
    -0.544921875,
    -4.66796875
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9893264770507812,
    0.008442666381597519,
    0.0020090744365006685,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.000221777125261724
   ],
   "p": [
    0.9895459413528442,
    0.00844453927129507,
    0.002009520074352622
   ],
   "p_oracle": [
    0.9895609617233276,
    0.008438709191977978,
    0.0020002825185656548
   ],
   "abstain": 0.000221777125261724,
   "abstain_oracle": 0.00022156724298838526,
   "abs_delta_abstain": 2.098822733387351e-07,
   "oracle_margin": 0.9811222553253174,
   "max_abs_delta_p": 1.5020370483398438e-05,
   "mean_abs_delta_p": 1.0029335195819536e-05,
   "max_abs_delta_raw_logits": 0.0048465728759765625,
   "mean_abs_delta_raw_logits": 0.0010368170915171504,
   "logits_min": -7.87109375,
   "logits_max": 4.09765625,
   "raw_logits_min": -7.87109375,
   "raw_logits_max": 4.09765625,
   "raw_logits_absmax": 7.87109375,
   "oracle_logits_min": -7.868396759033203,
   "oracle_logits_max": 4.100948810577393,
   "oracle_raw_logits_absmax": 7.868396759033203,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.26220699993427843,
   "seconds": 0.26220699993427843,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r12-direction",
   "request_id": "r12",
   "question_id": "direction",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 44,
   "nopts": 4,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.798828125,
    -2.513671875,
    -3.357421875,
    3.302734375,
    -1.912109375,
    -2.228515625,
    -2.83984375,
    -2.912109375,
    -3.337890625,
    -1.958984375,
    -1.8486328125,
    -2.767578125,
    -3.841796875,
    -1.39453125,
    -3.767578125,
    -4.828125,
    -4.49609375,
    -4.9921875,
    -2.81640625,
    -2.279296875,
    -3.814453125,
    -1.4267578125,
    -5.6015625,
    -5.49609375,
    -4.140625,
    -4.4765625,
    -2.869140625,
    -5.01171875,
    -3.619140625,
    -5.453125,
    -4.9453125,
    -4.33984375,
    -6.7421875,
    -3.466796875,
    -2.548828125,
    -5.05078125,
    -4.98828125,
    -6.078125,
    -3.83984375,
    -3.41796875,
    -6.82421875,
    -3.9140625,
    -5.01171875,
    -5.9921875,
    -3.6484375,
    -4.25390625,
    -4.85546875,
    -5.05859375,
    -5.421875,
    -3.615234375,
    -4.92578125,
    -4.109375,
    -2.673828125,
    -5.625,
    -2.34765625,
    -2.3671875,
    -4.48828125,
    -4.890625,
    -4.66015625,
    -5.921875,
    -7.2265625,
    -6.66796875,
    -7.75390625,
    -5.8125,
    -3.55859375,
    -7.00390625,
    -7.359375,
    -1.8408203125,
    -7.4140625,
    -4.46484375,
    -7.8515625,
    -5.23828125,
    -2.37890625,
    -5.26953125,
    -3.70703125,
    -5.9765625,
    -4.49609375,
    -6.875,
    -3.61328125,
    -5.3046875,
    -4.359375,
    -3.771484375,
    -2.33984375,
    -4.078125,
    -3.1875,
    -4.30859375,
    -2.63671875,
    -3.1015625,
    -3.154296875,
    -4.27734375,
    -3.669921875,
    -4.01953125,
    -3.904296875,
    -3.5859375,
    -4.8515625,
    -4.609375,
    -4.9296875,
    -3.51171875,
    -3.642578125,
    -3.837890625,
    -5.32421875,
    -5.0859375,
    -3.341796875,
    -3.9375,
    -4.59375,
    -5.375,
    -3.70703125,
    -4.76171875,
    -4.58203125,
    -5.3984375,
    -4.23046875,
    -4.92578125,
    -4.44140625,
    -5.8125,
    -4.8984375,
    -4.13671875,
    -4.01171875,
    -5.74609375,
    -3.9921875,
    -2.7734375,
    -2.66015625,
    -4.13671875,
    -3.23828125,
    -3.775390625,
    -1.994140625,
    -5.5,
    -5.2578125,
    -2.84765625,
    -3.564453125,
    -5.65625,
    -5.46484375,
    -2.4453125,
    -5.8671875,
    -6.08984375,
    -4.58203125,
    -4.53515625,
    -3.900390625,
    -4.734375,
    -6.0,
    -2.431640625,
    -3.25390625,
    -4.4140625,
    -3.5234375,
    -3.177734375,
    -4.140625,
    -4.796875,
    -5.89453125,
    -5.52734375,
    -5.67578125,
    -4.78125,
    -4.3046875,
    -2.794921875,
    -1.11328125,
    -3.958984375,
    -2.697265625,
    -1.5283203125,
    -1.9658203125,
    -1.763671875,
    -2.95703125,
    -0.99267578125,
    -2.45703125,
    -2.853515625,
    -1.6787109375,
    -1.04296875,
    -3.263671875,
    -2.005859375,
    -2.576171875,
    -2.470703125,
    -4.04296875,
    -2.16015625,
    -1.955078125,
    -2.34375,
    -2.232421875,
    -3.1015625,
    -1.08984375,
    -1.775390625,
    -3.435546875,
    -3.181640625,
    -2.609375,
    -3.05859375,
    -3.197265625,
    -2.552734375,
    -2.9921875,
    -1.755859375,
    -3.029296875,
    -2.1015625,
    -1.7294921875,
    -2.296875,
    -1.6552734375,
    -1.4931640625,
    -1.1025390625,
    -1.3203125,
    -2.59765625,
    -1.6552734375,
    -1.5537109375,
    -2.330078125,
    -1.6337890625,
    -1.50390625,
    -1.900390625,
    -1.1748046875,
    -1.5,
    -1.8388671875,
    -1.5263671875,
    -2.06640625,
    -2.1875,
    -1.3779296875,
    -2.03515625,
    -2.552734375,
    0.2340087890625,
    -2.244140625,
    -0.82958984375,
    -1.2626953125,
    -0.486083984375,
    -1.2412109375,
    -2.4375,
    -1.328125,
    -1.908203125,
    -2.166015625,
    -1.55859375,
    -1.8505859375,
    -1.1689453125,
    -1.435546875,
    -0.94091796875,
    -1.421875,
    0.15185546875,
    0.193359375,
    -0.8515625,
    0.74853515625,
    1.162109375,
    -0.197265625,
    -1.2421875,
    -0.6904296875,
    -0.264404296875,
    -1.1552734375,
    0.5615234375,
    0.7275390625,
    1.2646484375,
    -0.7255859375,
    -0.6513671875,
    -0.4814453125,
    0.62060546875,
    0.685546875,
    -0.783203125,
    0.288330078125,
    -0.08056640625,
    0.251708984375,
    0.162353515625,
    -0.290283203125,
    -0.11737060546875,
    1.6025390625,
    -0.0123748779296875,
    -0.48828125,
    -0.546875,
    1.1328125,
    0.0019159317016601562,
    -5.18359375
   ],
   "raw256_argmax": 3,
   "option_argmax": 3,
   "oracle_argmax": 3,
   "argmax_agrees": true,
   "p_full": [
    0.0031087114475667477,
    0.00406980887055397,
    0.0018340067472308874,
    0.9906608462333679,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00032670845394022763
   ],
   "p": [
    0.0031097272876650095,
    0.004071138799190521,
    0.0018346059368923306,
    0.9909845590591431
   ],
   "p_oracle": [
    0.003069420112296939,
    0.004049216862767935,
    0.0018126902868971229,
    0.9910686612129211
   ],
   "abstain": 0.00032670845394022763,
   "abstain_oracle": 0.00032492054742760956,
   "abs_delta_abstain": 1.7879065126180649e-06,
   "oracle_margin": 0.987019419670105,
   "max_abs_delta_p": 8.410215377807617e-05,
   "mean_abs_delta_p": 4.206172889098525e-05,
   "max_abs_delta_raw_logits": 0.009778976440429688,
   "mean_abs_delta_raw_logits": 0.002847052732249722,
   "logits_min": -7.8515625,
   "logits_max": 3.302734375,
   "raw_logits_min": -7.8515625,
   "raw_logits_max": 3.302734375,
   "raw_logits_absmax": 7.8515625,
   "oracle_logits_min": -7.86134147644043,
   "oracle_logits_max": 3.3078465461730957,
   "oracle_raw_logits_absmax": 7.86134147644043,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2500071249669418,
   "seconds": 0.2500071249669418,
   "full_vocab_argmax_id": 3,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r12-open",
   "request_id": "r12",
   "question_id": "open",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 36,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    2.607421875,
    -2.201171875,
    -1.953125,
    -2.34765625,
    -1.9345703125,
    -2.25390625,
    -2.677734375,
    -3.380859375,
    -2.859375,
    -1.912109375,
    -2.4921875,
    -3.11328125,
    -3.806640625,
    -2.3984375,
    -3.634765625,
    -4.25390625,
    -4.1171875,
    -4.88671875,
    -2.498046875,
    -2.994140625,
    -3.224609375,
    -1.4326171875,
    -4.8203125,
    -5.87890625,
    -3.185546875,
    -4.09375,
    -2.78125,
    -4.71484375,
    -3.82421875,
    -6.109375,
    -3.953125,
    -4.55859375,
    -5.76953125,
    -4.13671875,
    -2.484375,
    -4.40625,
    -4.80859375,
    -6.01953125,
    -4.296875,
    -3.259765625,
    -6.28515625,
    -4.0703125,
    -5.0078125,
    -5.83203125,
    -3.7578125,
    -4.3359375,
    -3.865234375,
    -4.76171875,
    -5.0625,
    -3.935546875,
    -5.29296875,
    -4.15234375,
    -3.201171875,
    -5.3125,
    -2.82421875,
    -3.369140625,
    -4.0234375,
    -4.625,
    -4.54296875,
    -6.20703125,
    -7.03515625,
    -6.765625,
    -7.7421875,
    -5.60546875,
    -3.583984375,
    -6.9765625,
    -7.25,
    -2.259765625,
    -7.11328125,
    -4.09375,
    -7.734375,
    -5.01953125,
    -2.966796875,
    -5.265625,
    -3.65234375,
    -5.9140625,
    -4.01171875,
    -6.7265625,
    -3.658203125,
    -5.65234375,
    -4.5703125,
    -3.716796875,
    -2.767578125,
    -4.375,
    -3.271484375,
    -4.18359375,
    -2.216796875,
    -3.83203125,
    -3.0078125,
    -4.39453125,
    -3.421875,
    -4.19140625,
    -4.07421875,
    -3.302734375,
    -4.33984375,
    -4.27734375,
    -4.6796875,
    -3.310546875,
    -2.912109375,
    -4.1015625,
    -5.25390625,
    -4.640625,
    -3.046875,
    -3.75390625,
    -4.328125,
    -5.48828125,
    -3.494140625,
    -4.89453125,
    -4.0859375,
    -5.93359375,
    -4.39453125,
    -4.6015625,
    -4.71875,
    -5.81640625,
    -4.58984375,
    -3.7421875,
    -4.55859375,
    -5.36328125,
    -3.751953125,
    -3.4140625,
    -3.05859375,
    -4.3515625,
    -3.06640625,
    -3.67578125,
    -2.22265625,
    -5.03515625,
    -5.37109375,
    -2.515625,
    -2.732421875,
    -5.265625,
    -5.75,
    -2.56640625,
    -5.46484375,
    -6.015625,
    -4.99609375,
    -3.94140625,
    -3.544921875,
    -4.59765625,
    -5.58984375,
    -2.623046875,
    -3.34765625,
    -4.00390625,
    -3.21875,
    -2.44140625,
    -4.671875,
    -4.94140625,
    -5.40625,
    -5.20703125,
    -5.046875,
    -4.390625,
    -4.3203125,
    -2.630859375,
    -0.931640625,
    -3.353515625,
    -2.89453125,
    -1.337890625,
    -1.7900390625,
    -1.8271484375,
    -2.958984375,
    -0.41162109375,
    -1.7177734375,
    -2.13671875,
    -1.61328125,
    -0.99072265625,
    -2.908203125,
    -1.8125,
    -2.599609375,
    -2.310546875,
    -3.654296875,
    -1.9130859375,
    -2.04296875,
    -2.44140625,
    -2.107421875,
    -2.91015625,
    -0.560546875,
    -1.8193359375,
    -3.626953125,
    -3.22265625,
    -2.6328125,
    -1.9658203125,
    -3.1015625,
    -2.197265625,
    -2.732421875,
    -1.9228515625,
    -2.255859375,
    -1.701171875,
    -1.4091796875,
    -1.376953125,
    -2.05859375,
    -2.26953125,
    -0.25,
    -1.6494140625,
    -2.25390625,
    -1.41015625,
    -1.580078125,
    -1.7080078125,
    -2.015625,
    -1.6220703125,
    -1.9501953125,
    -1.0537109375,
    -1.0732421875,
    -1.7890625,
    -1.630859375,
    -1.6669921875,
    -2.171875,
    -0.759765625,
    -2.15234375,
    -2.306640625,
    0.189208984375,
    -1.82421875,
    -1.1884765625,
    -0.9765625,
    0.08428955078125,
    -0.45458984375,
    -2.56640625,
    -0.76220703125,
    -1.5615234375,
    -1.7978515625,
    -1.087890625,
    -1.5634765625,
    -0.8720703125,
    -1.732421875,
    -0.69189453125,
    -1.0439453125,
    -0.28955078125,
    -0.116455078125,
    -0.62841796875,
    1.064453125,
    0.90380859375,
    -0.04559326171875,
    -0.414306640625,
    -0.67529296875,
    0.35888671875,
    -0.79248046875,
    0.291015625,
    0.5556640625,
    1.296875,
    -0.2061767578125,
    -0.0343017578125,
    -0.84619140625,
    0.46728515625,
    0.77294921875,
    -0.9013671875,
    -0.2177734375,
    -0.244140625,
    0.84033203125,
    0.52099609375,
    -0.77392578125,
    -0.64111328125,
    2.185546875,
    -0.2261962890625,
    -0.373291015625,
    -0.282958984375,
    0.34912109375,
    -0.403076171875,
    -5.73046875
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9913942217826843,
    0.008354835212230682,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0002508917241357267
   ],
   "p": [
    0.9916430711746216,
    0.008356932550668716
   ],
   "p_oracle": [
    0.9916988015174866,
    0.008301186375319958
   ],
   "abstain": 0.0002508917241357267,
   "abstain_oracle": 0.000250753277214244,
   "abs_delta_abstain": 1.3844692148268223e-07,
   "oracle_margin": 0.9833976030349731,
   "max_abs_delta_p": 5.57461753487587e-05,
   "mean_abs_delta_p": 5.5738259106874466e-05,
   "max_abs_delta_raw_logits": 0.004994392395019531,
   "mean_abs_delta_raw_logits": 0.0014897668734192848,
   "logits_min": -7.7421875,
   "logits_max": 2.607421875,
   "raw_logits_min": -7.7421875,
   "raw_logits_max": 2.607421875,
   "raw_logits_absmax": 7.7421875,
   "oracle_logits_min": -7.743186950683594,
   "oracle_logits_max": 2.6113667488098145,
   "oracle_raw_logits_absmax": 7.743186950683594,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.20216595800593495,
   "seconds": 0.20216595800593495,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r12-slope",
   "request_id": "r12",
   "question_id": "slope",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 50,
   "nopts": 3,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    4.18359375,
    -0.9267578125,
    -2.517578125,
    -2.94140625,
    -2.501953125,
    -2.798828125,
    -2.849609375,
    -4.07421875,
    -3.275390625,
    -2.142578125,
    -2.68359375,
    -3.619140625,
    -4.55859375,
    -2.74609375,
    -3.990234375,
    -4.18359375,
    -4.1328125,
    -4.609375,
    -2.806640625,
    -3.37890625,
    -3.279296875,
    -1.62109375,
    -5.109375,
    -6.32421875,
    -3.515625,
    -4.27734375,
    -2.564453125,
    -4.90625,
    -4.0859375,
    -6.4609375,
    -4.0625,
    -4.890625,
    -5.80078125,
    -4.42578125,
    -2.935546875,
    -4.6796875,
    -5.08984375,
    -6.08203125,
    -4.203125,
    -3.3828125,
    -6.67578125,
    -4.390625,
    -5.10546875,
    -6.06640625,
    -4.078125,
    -4.7734375,
    -3.96484375,
    -4.74609375,
    -5.4296875,
    -4.24609375,
    -5.48046875,
    -4.21875,
    -3.166015625,
    -5.4453125,
    -3.0625,
    -3.435546875,
    -4.0546875,
    -4.859375,
    -4.6171875,
    -6.45703125,
    -7.33984375,
    -6.796875,
    -8.078125,
    -5.6875,
    -3.837890625,
    -7.19140625,
    -7.60546875,
    -2.3046875,
    -7.33984375,
    -4.02734375,
    -7.9296875,
    -5.1875,
    -3.25,
    -5.48828125,
    -3.736328125,
    -6.15234375,
    -4.12890625,
    -6.9140625,
    -3.779296875,
    -5.859375,
    -4.609375,
    -3.962890625,
    -2.904296875,
    -4.6484375,
    -3.470703125,
    -4.28515625,
    -2.25390625,
    -4.03125,
    -3.03125,
    -4.8203125,
    -3.60546875,
    -4.41015625,
    -4.2109375,
    -3.60546875,
    -4.3828125,
    -4.2734375,
    -4.96484375,
    -3.529296875,
    -3.05859375,
    -4.15625,
    -5.625,
    -4.79296875,
    -3.2890625,
    -3.958984375,
    -4.515625,
    -5.85546875,
    -3.923828125,
    -5.31640625,
    -4.234375,
    -6.09765625,
    -4.55078125,
    -4.65625,
    -5.09765625,
    -6.125,
    -4.9296875,
    -4.0,
    -4.78125,
    -5.71484375,
    -3.85546875,
    -3.716796875,
    -3.291015625,
    -4.36328125,
    -3.193359375,
    -3.685546875,
    -2.431640625,
    -5.3203125,
    -5.6875,
    -2.76953125,
    -2.87109375,
    -5.66015625,
    -6.0703125,
    -2.814453125,
    -5.5234375,
    -6.24609375,
    -4.875,
    -3.97265625,
    -3.677734375,
    -4.94921875,
    -6.05859375,
    -3.060546875,
    -3.599609375,
    -4.2109375,
    -3.29296875,
    -2.310546875,
    -5.07421875,
    -5.1953125,
    -5.828125,
    -5.359375,
    -4.97265625,
    -4.625,
    -4.4296875,
    -2.794921875,
    -1.033203125,
    -3.658203125,
    -2.888671875,
    -1.5947265625,
    -1.802734375,
    -1.984375,
    -2.962890625,
    -0.5029296875,
    -1.7314453125,
    -2.404296875,
    -1.7734375,
    -0.9609375,
    -2.8515625,
    -1.759765625,
    -2.759765625,
    -2.5546875,
    -3.65234375,
    -1.814453125,
    -2.287109375,
    -2.556640625,
    -2.375,
    -3.017578125,
    -0.75634765625,
    -2.140625,
    -3.87109375,
    -3.369140625,
    -2.861328125,
    -2.35546875,
    -3.20703125,
    -2.716796875,
    -2.837890625,
    -2.25,
    -2.4921875,
    -1.9306640625,
    -1.1806640625,
    -1.4482421875,
    -2.38671875,
    -2.73046875,
    -0.53564453125,
    -1.7734375,
    -2.55078125,
    -1.705078125,
    -1.810546875,
    -1.6689453125,
    -2.19140625,
    -1.919921875,
    -2.16796875,
    -1.0634765625,
    -1.1435546875,
    -1.8232421875,
    -1.4443359375,
    -1.94140625,
    -2.357421875,
    -1.1162109375,
    -2.37890625,
    -2.55859375,
    0.01178741455078125,
    -2.12890625,
    -1.4873046875,
    -1.203125,
    0.05224609375,
    -0.67529296875,
    -2.6796875,
    -0.9990234375,
    -1.7509765625,
    -1.90625,
    -1.271484375,
    -1.9609375,
    -1.03515625,
    -1.7841796875,
    -0.72119140625,
    -1.146484375,
    -0.292236328125,
    -0.17724609375,
    -0.68359375,
    1.22265625,
    0.9189453125,
    -0.2083740234375,
    -0.4248046875,
    -0.66455078125,
    0.619140625,
    -0.71142578125,
    0.314453125,
    0.54052734375,
    1.3076171875,
    -0.2208251953125,
    0.341552734375,
    -0.89013671875,
    0.338623046875,
    0.72265625,
    -0.8076171875,
    -0.253662109375,
    -0.13134765625,
    0.7138671875,
    0.60986328125,
    -0.4384765625,
    -0.85595703125,
    2.271484375,
    -0.1922607421875,
    -0.235595703125,
    -0.1754150390625,
    0.2666015625,
    -0.43408203125,
    -4.48046875
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9907628893852234,
    0.007385153789073229,
    0.0016071187565103173,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0002448109444230795
   ],
   "p": [
    0.991005539894104,
    0.007386962417513132,
    0.0016075123567134142
   ],
   "p_oracle": [
    0.9910877346992493,
    0.007320776116102934,
    0.0015914562391117215
   ],
   "abstain": 0.0002448109444230795,
   "abstain_oracle": 0.0002448532613925636,
   "abs_delta_abstain": 4.2316969484090805e-08,
   "oracle_margin": 0.9837669730186462,
   "max_abs_delta_p": 8.219480514526367e-05,
   "mean_abs_delta_p": 5.481240805238485e-05,
   "max_abs_delta_raw_logits": 0.007946491241455078,
   "mean_abs_delta_raw_logits": 0.002350806404137984,
   "logits_min": -8.078125,
   "logits_max": 4.18359375,
   "raw_logits_min": -8.078125,
   "raw_logits_max": 4.18359375,
   "raw_logits_absmax": 8.078125,
   "oracle_logits_min": -8.079290390014648,
   "oracle_logits_max": 4.1894683837890625,
   "oracle_raw_logits_absmax": 8.079290390014648,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2814517499646172,
   "seconds": 0.2814517499646172,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r13-door",
   "request_id": "r13",
   "question_id": "door",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 66,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.3203125,
    3.43359375,
    -2.9453125,
    -2.6484375,
    -1.9052734375,
    -2.890625,
    -2.923828125,
    -3.830078125,
    -4.02734375,
    -3.01953125,
    -3.251953125,
    -4.234375,
    -5.69921875,
    -3.646484375,
    -4.5234375,
    -5.00390625,
    -4.05078125,
    -4.35546875,
    -3.205078125,
    -3.248046875,
    -3.48828125,
    -1.994140625,
    -5.1015625,
    -7.26953125,
    -4.21875,
    -5.48046875,
    -2.734375,
    -5.82421875,
    -4.49609375,
    -7.22265625,
    -4.6796875,
    -4.9765625,
    -6.43359375,
    -5.2734375,
    -3.19921875,
    -5.53515625,
    -5.140625,
    -5.8984375,
    -4.265625,
    -3.2421875,
    -7.3359375,
    -4.5,
    -4.97265625,
    -6.26953125,
    -4.74609375,
    -5.28125,
    -4.671875,
    -4.70703125,
    -5.203125,
    -4.74609375,
    -5.99609375,
    -4.75,
    -2.890625,
    -5.1640625,
    -2.67578125,
    -3.1953125,
    -3.7734375,
    -4.78125,
    -4.8828125,
    -5.91015625,
    -7.8046875,
    -6.6953125,
    -7.84375,
    -6.7890625,
    -3.8203125,
    -7.58984375,
    -7.8046875,
    -2.345703125,
    -7.93359375,
    -4.16796875,
    -8.359375,
    -5.5703125,
    -2.861328125,
    -6.25390625,
    -3.87109375,
    -6.65625,
    -4.52734375,
    -7.125,
    -4.30078125,
    -5.9140625,
    -4.37890625,
    -4.37109375,
    -2.8671875,
    -4.296875,
    -3.65625,
    -4.04296875,
    -2.732421875,
    -3.9453125,
    -3.09375,
    -5.328125,
    -3.626953125,
    -4.26171875,
    -4.3515625,
    -4.24609375,
    -4.78515625,
    -5.00390625,
    -5.26171875,
    -4.12890625,
    -4.3984375,
    -4.2265625,
    -6.2734375,
    -4.83203125,
    -3.615234375,
    -4.45703125,
    -4.69140625,
    -6.36328125,
    -4.59375,
    -5.8515625,
    -4.34765625,
    -5.8046875,
    -4.4609375,
    -5.1015625,
    -5.5078125,
    -6.93359375,
    -4.99609375,
    -4.32421875,
    -4.79296875,
    -6.17578125,
    -4.58203125,
    -3.8046875,
    -3.5234375,
    -4.01171875,
    -3.40625,
    -3.962890625,
    -2.64453125,
    -6.14453125,
    -6.04296875,
    -3.38671875,
    -3.220703125,
    -5.69921875,
    -5.88671875,
    -4.0078125,
    -5.890625,
    -6.1484375,
    -5.234375,
    -4.87890625,
    -4.3203125,
    -5.1328125,
    -6.13671875,
    -3.15234375,
    -3.859375,
    -4.2265625,
    -3.9765625,
    -3.30859375,
    -5.12890625,
    -5.6171875,
    -6.36328125,
    -5.41796875,
    -4.984375,
    -4.68359375,
    -4.92578125,
    -3.36328125,
    -1.1689453125,
    -3.61328125,
    -2.75,
    -1.76953125,
    -2.109375,
    -2.18359375,
    -3.43359375,
    -0.57958984375,
    -2.3203125,
    -2.1171875,
    -1.765625,
    -1.234375,
    -2.6875,
    -2.578125,
    -2.87890625,
    -1.9755859375,
    -3.46875,
    -2.134765625,
    -2.5234375,
    -2.625,
    -2.607421875,
    -3.21875,
    -1.0947265625,
    -2.02734375,
    -3.34765625,
    -3.357421875,
    -3.1796875,
    -3.6015625,
    -3.484375,
    -3.45703125,
    -2.828125,
    -2.251953125,
    -2.63671875,
    -1.7060546875,
    -1.328125,
    -1.9697265625,
    -1.9912109375,
    -2.23046875,
    -1.15625,
    -1.90625,
    -3.013671875,
    -1.7783203125,
    -1.5419921875,
    -2.142578125,
    -2.140625,
    -1.90234375,
    -1.828125,
    -1.4892578125,
    -1.2412109375,
    -2.0859375,
    -1.857421875,
    -1.9248046875,
    -2.572265625,
    -1.3212890625,
    -2.783203125,
    -2.705078125,
    -0.436767578125,
    -2.4609375,
    -1.033203125,
    -1.3056640625,
    -0.8984375,
    -1.623046875,
    -2.515625,
    -1.3564453125,
    -1.88671875,
    -2.3671875,
    -1.8779296875,
    -1.9912109375,
    -1.3916015625,
    -1.8349609375,
    -1.431640625,
    -1.42578125,
    -0.2484130859375,
    0.12744140625,
    -0.3349609375,
    0.9013671875,
    1.0546875,
    -0.01293182373046875,
    -0.87109375,
    -0.662109375,
    0.159912109375,
    -0.802734375,
    0.58544921875,
    1.01953125,
    1.5068359375,
    -0.5849609375,
    0.071533203125,
    -0.7109375,
    0.30810546875,
    0.84521484375,
    -0.875,
    -0.2205810546875,
    0.1748046875,
    0.67138671875,
    0.2144775390625,
    -0.3623046875,
    -0.96044921875,
    1.5390625,
    0.008941650390625,
    -0.69921875,
    -0.1273193359375,
    0.468994140625,
    -0.51123046875,
    -4.52734375
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.004326396621763706,
    0.992738664150238,
    0.0023971865884959698,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0005378078203648329
   ],
   "p": [
    0.004328724462538958,
    0.9932727813720703,
    0.002398476470261812
   ],
   "p_oracle": [
    0.0043136803433299065,
    0.9933029413223267,
    0.002383379964157939
   ],
   "abstain": 0.0005378078203648329,
   "abstain_oracle": 0.0005352262523956597,
   "abs_delta_abstain": 2.581567969173193e-06,
   "oracle_margin": 0.9889892339706421,
   "max_abs_delta_p": 3.0159950256347656e-05,
   "mean_abs_delta_p": 2.0100191856424015e-05,
   "max_abs_delta_raw_logits": 0.010681629180908203,
   "mean_abs_delta_raw_logits": 0.003950562735553831,
   "logits_min": -8.359375,
   "logits_max": 3.43359375,
   "raw_logits_min": -8.359375,
   "raw_logits_max": 3.43359375,
   "raw_logits_absmax": 8.359375,
   "oracle_logits_min": -8.369766235351562,
   "oracle_logits_max": 3.435242176055908,
   "oracle_raw_logits_absmax": 8.369766235351562,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3696989578893408,
   "seconds": 0.3696989578893408,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r13-light",
   "request_id": "r13",
   "question_id": "light",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 65,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    -3.25390625,
    1.962890625,
    -1.697265625,
    -1.6708984375,
    -1.7431640625,
    -2.833984375,
    -2.560546875,
    -3.1796875,
    -3.421875,
    -2.41796875,
    -2.478515625,
    -3.638671875,
    -4.63671875,
    -2.3046875,
    -3.38671875,
    -4.53125,
    -4.0390625,
    -4.86328125,
    -2.9921875,
    -2.81640625,
    -3.439453125,
    -1.59375,
    -4.99609375,
    -6.81640625,
    -3.91796875,
    -4.96875,
    -2.560546875,
    -5.55078125,
    -3.98046875,
    -6.38671875,
    -4.50390625,
    -4.5625,
    -6.09765625,
    -4.57421875,
    -2.873046875,
    -5.1328125,
    -5.16796875,
    -5.8203125,
    -4.21484375,
    -3.181640625,
    -6.9609375,
    -4.20703125,
    -4.7265625,
    -6.10546875,
    -4.06640625,
    -4.8046875,
    -4.26171875,
    -4.5078125,
    -4.95703125,
    -4.41015625,
    -5.50390625,
    -4.43359375,
    -2.642578125,
    -4.8046875,
    -2.45703125,
    -3.087890625,
    -3.60546875,
    -4.70703125,
    -4.53125,
    -5.40234375,
    -7.29296875,
    -6.39453125,
    -7.390625,
    -6.30859375,
    -3.44921875,
    -7.33203125,
    -7.4765625,
    -2.287109375,
    -7.60546875,
    -4.32421875,
    -7.9375,
    -5.0859375,
    -2.4921875,
    -5.953125,
    -3.751953125,
    -6.125,
    -4.3984375,
    -6.6953125,
    -3.9453125,
    -5.48046875,
    -4.14453125,
    -3.958984375,
    -2.673828125,
    -3.966796875,
    -3.33203125,
    -4.09375,
    -2.431640625,
    -3.48828125,
    -2.80078125,
    -4.80078125,
    -3.5546875,
    -3.95703125,
    -3.68359375,
    -4.14453125,
    -4.66796875,
    -4.80859375,
    -4.87890625,
    -3.85546875,
    -3.748046875,
    -4.0546875,
    -5.6875,
    -4.765625,
    -3.380859375,
    -3.904296875,
    -4.23046875,
    -5.8515625,
    -3.921875,
    -5.3984375,
    -4.19921875,
    -5.203125,
    -4.30859375,
    -4.66015625,
    -4.96484375,
    -6.34375,
    -4.71484375,
    -3.91796875,
    -4.4921875,
    -5.74609375,
    -4.2734375,
    -3.357421875,
    -3.259765625,
    -3.93359375,
    -3.431640625,
    -3.5703125,
    -2.505859375,
    -5.80078125,
    -5.2890625,
    -3.18359375,
    -2.884765625,
    -5.328125,
    -5.71875,
    -3.28515625,
    -5.48046875,
    -5.9375,
    -5.16015625,
    -4.5625,
    -3.705078125,
    -4.84375,
    -5.62890625,
    -2.83203125,
    -3.515625,
    -4.01953125,
    -3.72265625,
    -3.310546875,
    -4.7734375,
    -5.05078125,
    -5.83984375,
    -5.1171875,
    -5.03125,
    -4.37109375,
    -4.5625,
    -3.14453125,
    -0.96435546875,
    -3.29296875,
    -2.494140625,
    -1.4658203125,
    -1.7744140625,
    -2.21484375,
    -3.302734375,
    -0.36767578125,
    -2.181640625,
    -2.001953125,
    -1.5986328125,
    -1.1943359375,
    -2.693359375,
    -2.453125,
    -2.693359375,
    -1.6181640625,
    -3.201171875,
    -2.048828125,
    -2.021484375,
    -2.423828125,
    -2.30859375,
    -2.978515625,
    -0.95068359375,
    -1.8310546875,
    -2.900390625,
    -3.236328125,
    -2.923828125,
    -3.330078125,
    -3.115234375,
    -2.998046875,
    -2.87890625,
    -1.98046875,
    -2.31640625,
    -1.5673828125,
    -1.4248046875,
    -1.8056640625,
    -1.6357421875,
    -2.1015625,
    -0.9287109375,
    -1.4453125,
    -2.72265625,
    -1.435546875,
    -1.525390625,
    -2.099609375,
    -2.037109375,
    -1.8193359375,
    -1.650390625,
    -1.21875,
    -1.1650390625,
    -1.771484375,
    -1.6845703125,
    -1.4755859375,
    -2.396484375,
    -0.93603515625,
    -2.56640625,
    -2.5859375,
    -0.11590576171875,
    -2.16796875,
    -0.7158203125,
    -1.0341796875,
    -0.6630859375,
    -1.0556640625,
    -2.48046875,
    -1.1337890625,
    -1.71484375,
    -2.216796875,
    -1.7109375,
    -1.66015625,
    -1.2607421875,
    -1.75390625,
    -1.3701171875,
    -1.4189453125,
    -0.2100830078125,
    0.332763671875,
    -0.0616455078125,
    0.81982421875,
    0.95263671875,
    -0.10809326171875,
    -0.96728515625,
    -0.70166015625,
    0.046722412109375,
    -0.79931640625,
    0.5283203125,
    1.15625,
    1.4560546875,
    -0.626953125,
    -0.051910400390625,
    -0.86474609375,
    0.64697265625,
    0.77294921875,
    -0.80517578125,
    -0.2412109375,
    -0.10198974609375,
    0.49951171875,
    0.1279296875,
    -0.47802734375,
    -0.9541015625,
    1.5927734375,
    0.053253173828125,
    -0.8349609375,
    0.00711822509765625,
    0.55029296875,
    -0.58251953125,
    -6.125
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.005585081409662962,
    0.9940925240516663,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003224858082830906
   ],
   "p": [
    0.005586882587522268,
    0.9944131374359131
   ],
   "p_oracle": [
    0.005548183340579271,
    0.9944518208503723
   ],
   "abstain": 0.0003224858082830906,
   "abstain_oracle": 0.0003229013236705214,
   "abs_delta_abstain": 4.155153874307871e-07,
   "oracle_margin": 0.9889036417007446,
   "max_abs_delta_p": 3.869924694299698e-05,
   "mean_abs_delta_p": 3.869133070111275e-05,
   "max_abs_delta_raw_logits": 0.008025169372558594,
   "mean_abs_delta_raw_logits": 0.002581561799161136,
   "logits_min": -7.9375,
   "logits_max": 1.962890625,
   "raw_logits_min": -7.9375,
   "raw_logits_max": 1.962890625,
   "raw_logits_absmax": 7.9375,
   "oracle_logits_min": -7.945124626159668,
   "oracle_logits_max": 1.9632949829101562,
   "oracle_raw_logits_absmax": 7.945124626159668,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.36413991695735604,
   "seconds": 0.36413991695735604,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r13-temperature",
   "request_id": "r13",
   "question_id": "temperature",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 83,
   "nopts": 3,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    -1.9892578125,
    3.654296875,
    -2.48046875,
    -2.634765625,
    -1.7802734375,
    -2.5859375,
    -2.7421875,
    -3.986328125,
    -3.681640625,
    -2.673828125,
    -3.42578125,
    -4.38671875,
    -5.5390625,
    -3.568359375,
    -4.48828125,
    -5.08984375,
    -3.951171875,
    -4.375,
    -3.056640625,
    -3.142578125,
    -3.29296875,
    -1.7041015625,
    -5.08984375,
    -7.2578125,
    -4.16796875,
    -5.37890625,
    -2.59375,
    -5.95703125,
    -4.46875,
    -7.3125,
    -4.7265625,
    -5.2265625,
    -6.52734375,
    -5.21484375,
    -3.0,
    -5.51953125,
    -5.1796875,
    -6.03125,
    -4.3203125,
    -3.25390625,
    -7.44921875,
    -4.55078125,
    -5.03125,
    -6.296875,
    -4.78515625,
    -5.21484375,
    -4.75390625,
    -4.6796875,
    -5.29296875,
    -4.671875,
    -6.01953125,
    -4.7578125,
    -2.740234375,
    -5.2421875,
    -2.6875,
    -3.306640625,
    -3.7890625,
    -4.9765625,
    -4.95703125,
    -6.19140625,
    -7.80859375,
    -6.7265625,
    -7.91796875,
    -6.84765625,
    -3.67578125,
    -7.6171875,
    -7.75390625,
    -2.423828125,
    -7.8828125,
    -4.23046875,
    -8.390625,
    -5.5625,
    -2.904296875,
    -6.2734375,
    -3.908203125,
    -6.59375,
    -4.3828125,
    -7.171875,
    -4.234375,
    -5.8671875,
    -4.44921875,
    -4.29296875,
    -2.83203125,
    -4.18359375,
    -3.62109375,
    -4.11328125,
    -2.771484375,
    -3.955078125,
    -2.921875,
    -5.30078125,
    -3.654296875,
    -4.24609375,
    -4.375,
    -4.25390625,
    -4.765625,
    -4.96484375,
    -5.2578125,
    -4.12890625,
    -4.4296875,
    -4.19921875,
    -6.29296875,
    -4.78125,
    -3.435546875,
    -4.39453125,
    -4.56640625,
    -6.39453125,
    -4.62109375,
    -5.94921875,
    -4.37890625,
    -5.7421875,
    -4.42578125,
    -4.96484375,
    -5.6796875,
    -7.15234375,
    -4.9375,
    -4.38671875,
    -4.86328125,
    -6.2265625,
    -4.421875,
    -3.720703125,
    -3.576171875,
    -4.12890625,
    -3.564453125,
    -3.90234375,
    -2.6875,
    -6.140625,
    -5.99609375,
    -3.48828125,
    -3.205078125,
    -5.9140625,
    -6.05078125,
    -3.853515625,
    -5.83984375,
    -6.203125,
    -5.26953125,
    -4.69140625,
    -4.23046875,
    -5.31640625,
    -6.09765625,
    -3.244140625,
    -3.84375,
    -4.13671875,
    -4.01953125,
    -3.228515625,
    -5.21484375,
    -5.75390625,
    -6.4296875,
    -5.43359375,
    -4.9609375,
    -4.90625,
    -4.953125,
    -3.544921875,
    -1.134765625,
    -3.779296875,
    -2.841796875,
    -1.7177734375,
    -2.21484375,
    -2.408203125,
    -3.58203125,
    -0.436279296875,
    -2.28125,
    -2.376953125,
    -1.779296875,
    -1.3203125,
    -2.849609375,
    -2.5234375,
    -2.876953125,
    -1.896484375,
    -3.423828125,
    -2.0625,
    -2.37109375,
    -2.59375,
    -2.650390625,
    -3.3046875,
    -1.17578125,
    -2.1015625,
    -3.40234375,
    -3.388671875,
    -3.056640625,
    -3.54296875,
    -3.330078125,
    -3.40234375,
    -2.8515625,
    -2.078125,
    -2.59375,
    -1.5791015625,
    -1.259765625,
    -1.927734375,
    -1.8525390625,
    -2.326171875,
    -1.0087890625,
    -1.716796875,
    -2.931640625,
    -1.66015625,
    -1.736328125,
    -2.15234375,
    -2.033203125,
    -1.884765625,
    -1.716796875,
    -1.455078125,
    -1.1142578125,
    -1.9677734375,
    -1.8154296875,
    -1.9443359375,
    -2.623046875,
    -1.19140625,
    -2.904296875,
    -2.669921875,
    -0.34375,
    -2.50390625,
    -0.98876953125,
    -1.1611328125,
    -0.904296875,
    -1.6708984375,
    -2.447265625,
    -1.2783203125,
    -1.865234375,
    -2.439453125,
    -1.9443359375,
    -1.951171875,
    -1.091796875,
    -1.8291015625,
    -1.3525390625,
    -1.625,
    -0.3291015625,
    0.20654296875,
    -0.2265625,
    0.86962890625,
    1.158203125,
    -0.0125885009765625,
    -0.953125,
    -0.7236328125,
    0.2529296875,
    -0.7509765625,
    0.56103515625,
    1.0595703125,
    1.5498046875,
    -0.5185546875,
    0.1580810546875,
    -0.81201171875,
    0.3515625,
    0.7939453125,
    -0.85498046875,
    -0.2626953125,
    0.167724609375,
    0.74853515625,
    0.2171630859375,
    -0.3955078125,
    -1.0400390625,
    1.580078125,
    0.09991455078125,
    -0.775390625,
    -0.07330322265625,
    0.4228515625,
    -0.505859375,
    -5.17578125
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.004437779542058706,
    0.992581844329834,
    0.002771139144897461,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00020917505025863647
   ],
   "p": [
    0.004438708070665598,
    0.9927895665168762,
    0.0027717191260308027
   ],
   "p_oracle": [
    0.004428014624863863,
    0.9928196668624878,
    0.002752336673438549
   ],
   "abstain": 0.00020917505025863647,
   "abstain_oracle": 0.00020866226986981928,
   "abs_delta_abstain": 5.127803888171911e-07,
   "oracle_margin": 0.988391637802124,
   "max_abs_delta_p": 3.0100345611572266e-05,
   "mean_abs_delta_p": 2.0058748001853626e-05,
   "max_abs_delta_raw_logits": 0.010114192962646484,
   "mean_abs_delta_raw_logits": 0.0029841608193237334,
   "logits_min": -8.390625,
   "logits_max": 3.654296875,
   "raw_logits_min": -8.390625,
   "raw_logits_max": 3.654296875,
   "raw_logits_absmax": 8.390625,
   "oracle_logits_min": -8.394624710083008,
   "oracle_logits_max": 3.6570920944213867,
   "oracle_raw_logits_absmax": 8.394624710083008,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4652311250101775,
   "seconds": 0.4652311250101775,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r14-color",
   "request_id": "r14",
   "question_id": "color",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 71,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    3.509765625,
    -2.45703125,
    -2.474609375,
    -2.662109375,
    -2.404296875,
    -2.51171875,
    -2.673828125,
    -3.837890625,
    -3.416015625,
    -2.185546875,
    -2.595703125,
    -3.341796875,
    -4.609375,
    -2.71484375,
    -3.888671875,
    -3.904296875,
    -4.04296875,
    -4.609375,
    -2.890625,
    -3.236328125,
    -3.203125,
    -1.86328125,
    -5.2109375,
    -5.99609375,
    -3.83203125,
    -4.28515625,
    -2.9453125,
    -4.75,
    -4.08203125,
    -6.15625,
    -3.98828125,
    -4.71484375,
    -6.00390625,
    -4.23046875,
    -3.146484375,
    -4.65625,
    -5.1328125,
    -6.19140625,
    -4.30859375,
    -3.296875,
    -6.2890625,
    -4.42578125,
    -5.1875,
    -6.0703125,
    -3.875,
    -4.64453125,
    -4.1171875,
    -4.765625,
    -5.30859375,
    -4.125,
    -5.1796875,
    -4.07421875,
    -3.17578125,
    -5.56640625,
    -3.140625,
    -3.5,
    -4.0703125,
    -4.83984375,
    -4.65234375,
    -6.17578125,
    -7.39453125,
    -6.88671875,
    -7.97265625,
    -5.609375,
    -3.912109375,
    -7.18359375,
    -7.55859375,
    -2.51953125,
    -7.23828125,
    -3.96875,
    -7.9453125,
    -5.30078125,
    -3.125,
    -5.4375,
    -3.67578125,
    -6.14453125,
    -4.234375,
    -6.8828125,
    -3.873046875,
    -6.0546875,
    -4.4921875,
    -4.0859375,
    -2.984375,
    -4.421875,
    -3.359375,
    -4.48828125,
    -2.416015625,
    -4.1015625,
    -3.109375,
    -4.6015625,
    -3.77734375,
    -4.2734375,
    -4.2890625,
    -3.5625,
    -4.38671875,
    -4.375,
    -4.94921875,
    -3.3125,
    -2.89453125,
    -4.203125,
    -5.5625,
    -4.68359375,
    -3.2734375,
    -3.97265625,
    -4.59375,
    -5.66796875,
    -3.72265625,
    -5.1015625,
    -4.3359375,
    -5.91796875,
    -4.43359375,
    -4.7578125,
    -5.1484375,
    -5.96875,
    -4.8515625,
    -3.75,
    -4.65625,
    -5.6484375,
    -3.8359375,
    -3.658203125,
    -3.130859375,
    -4.3125,
    -3.0859375,
    -3.685546875,
    -2.390625,
    -5.234375,
    -5.765625,
    -2.541015625,
    -2.90625,
    -5.67578125,
    -5.96875,
    -2.7734375,
    -5.69921875,
    -6.42578125,
    -4.96875,
    -4.0625,
    -3.763671875,
    -4.66015625,
    -6.09375,
    -2.88671875,
    -3.509765625,
    -4.23828125,
    -3.26953125,
    -2.478515625,
    -4.96875,
    -5.08984375,
    -5.75390625,
    -5.5,
    -5.24609375,
    -4.609375,
    -4.58984375,
    -2.7890625,
    -1.0,
    -3.453125,
    -2.67578125,
    -1.7490234375,
    -1.724609375,
    -1.9384765625,
    -2.857421875,
    -0.5185546875,
    -1.7607421875,
    -1.9599609375,
    -1.6162109375,
    -0.9150390625,
    -2.884765625,
    -1.7216796875,
    -2.6796875,
    -2.416015625,
    -3.693359375,
    -1.9013671875,
    -2.330078125,
    -2.525390625,
    -2.34375,
    -3.04296875,
    -0.72119140625,
    -1.9775390625,
    -3.673828125,
    -3.474609375,
    -3.005859375,
    -2.2421875,
    -3.26953125,
    -2.412109375,
    -2.748046875,
    -2.1171875,
    -2.513671875,
    -2.02734375,
    -1.375,
    -1.4501953125,
    -2.24609375,
    -2.765625,
    -0.56103515625,
    -1.8583984375,
    -2.576171875,
    -1.5888671875,
    -1.78125,
    -1.845703125,
    -2.060546875,
    -1.763671875,
    -2.22265625,
    -1.0390625,
    -1.1591796875,
    -1.9208984375,
    -1.5234375,
    -1.8984375,
    -2.43359375,
    -1.2197265625,
    -2.16796875,
    -2.474609375,
    0.16455078125,
    -2.00390625,
    -1.4814453125,
    -1.1904296875,
    0.06365966796875,
    -0.69580078125,
    -2.865234375,
    -0.85693359375,
    -1.6181640625,
    -1.8525390625,
    -1.1279296875,
    -2.029296875,
    -1.2978515625,
    -1.76953125,
    -0.69873046875,
    -1.0244140625,
    -0.145263671875,
    -0.1802978515625,
    -0.509765625,
    0.97802734375,
    0.7958984375,
    -0.26953125,
    -0.364501953125,
    -0.6044921875,
    0.59423828125,
    -0.6904296875,
    0.321533203125,
    0.7529296875,
    1.1328125,
    -0.1259765625,
    0.0782470703125,
    -0.72314453125,
    0.307373046875,
    0.73291015625,
    -0.79443359375,
    -0.399169921875,
    -0.343017578125,
    0.69970703125,
    0.6650390625,
    -0.59423828125,
    -0.67333984375,
    2.236328125,
    -0.263916015625,
    -0.184814453125,
    -0.28466796875,
    0.21240234375,
    -0.486083984375,
    -4.203125
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9923054575920105,
    0.003536652307957411,
    0.003478406462818384,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0006795263034291565
   ],
   "p": [
    0.9929801821708679,
    0.003539056982845068,
    0.00348077155649662
   ],
   "p_oracle": [
    0.9929444193840027,
    0.0035478509962558746,
    0.0035077447537332773
   ],
   "abstain": 0.0006795263034291565,
   "abstain_oracle": 0.0006824982701800764,
   "abs_delta_abstain": 2.971966750919819e-06,
   "oracle_margin": 0.9893965721130371,
   "max_abs_delta_p": 3.5762786865234375e-05,
   "mean_abs_delta_p": 2.3843332504232723e-05,
   "max_abs_delta_raw_logits": 0.007548809051513672,
   "mean_abs_delta_raw_logits": 0.0018511592643335462,
   "logits_min": -7.97265625,
   "logits_max": 3.509765625,
   "raw_logits_min": -7.97265625,
   "raw_logits_max": 3.509765625,
   "raw_logits_absmax": 7.97265625,
   "oracle_logits_min": -7.96762752532959,
   "oracle_logits_max": 3.5039243698120117,
   "oracle_raw_logits_absmax": 7.96762752532959,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.39725270902272314,
   "seconds": 0.39725270902272314,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r14-wet",
   "request_id": "r14",
   "question_id": "wet",
   "type": "noul",
   "qtype_index": 2,
   "kind": "slot",
   "tokens": 61,
   "nopts": 2,
   "temperature": 1.0067671537399292,
   "raw_logits": [
    2.076171875,
    -2.05859375,
    -1.783203125,
    -2.45703125,
    -2.306640625,
    -2.47265625,
    -2.83203125,
    -3.23828125,
    -2.955078125,
    -2.24609375,
    -2.24609375,
    -3.0703125,
    -3.802734375,
    -2.373046875,
    -3.361328125,
    -4.0078125,
    -3.9765625,
    -4.640625,
    -2.47265625,
    -3.0703125,
    -3.345703125,
    -1.4609375,
    -4.8125,
    -5.70703125,
    -3.1796875,
    -4.0234375,
    -2.783203125,
    -4.91015625,
    -3.666015625,
    -5.9375,
    -3.84765625,
    -4.609375,
    -5.76953125,
    -3.994140625,
    -2.80859375,
    -4.515625,
    -4.8984375,
    -5.94140625,
    -4.34375,
    -3.07421875,
    -6.1953125,
    -4.15625,
    -4.7890625,
    -5.77734375,
    -3.587890625,
    -4.16015625,
    -3.912109375,
    -4.70703125,
    -4.6796875,
    -4.08984375,
    -5.16015625,
    -4.078125,
    -3.314453125,
    -5.23046875,
    -2.890625,
    -3.31640625,
    -3.80078125,
    -4.58203125,
    -4.421875,
    -5.9609375,
    -6.91015625,
    -6.74609375,
    -7.76171875,
    -5.55078125,
    -3.560546875,
    -6.9609375,
    -7.2265625,
    -2.1171875,
    -7.04296875,
    -3.990234375,
    -7.46484375,
    -4.9921875,
    -2.8515625,
    -5.2109375,
    -3.47265625,
    -5.9453125,
    -4.06640625,
    -6.55859375,
    -3.654296875,
    -5.71484375,
    -4.36328125,
    -3.671875,
    -2.783203125,
    -4.2265625,
    -3.130859375,
    -4.1640625,
    -2.265625,
    -3.818359375,
    -3.021484375,
    -4.25390625,
    -3.30859375,
    -3.845703125,
    -3.8203125,
    -3.318359375,
    -4.27734375,
    -4.1640625,
    -4.59375,
    -3.16015625,
    -2.677734375,
    -4.0859375,
    -5.23828125,
    -4.6640625,
    -3.111328125,
    -3.63671875,
    -4.3046875,
    -5.53125,
    -3.3515625,
    -4.859375,
    -3.99609375,
    -5.65234375,
    -4.4609375,
    -4.60546875,
    -4.52734375,
    -5.5546875,
    -4.52734375,
    -3.66796875,
    -4.48828125,
    -5.421875,
    -3.7890625,
    -3.46875,
    -3.0,
    -4.125,
    -3.03125,
    -3.5625,
    -2.03125,
    -4.984375,
    -5.30859375,
    -2.642578125,
    -2.76171875,
    -5.15625,
    -5.703125,
    -2.6640625,
    -5.39453125,
    -6.04296875,
    -4.95703125,
    -4.07421875,
    -3.521484375,
    -4.65234375,
    -5.5703125,
    -2.478515625,
    -3.244140625,
    -3.990234375,
    -3.236328125,
    -2.525390625,
    -4.625,
    -5.1015625,
    -5.37109375,
    -5.25390625,
    -5.11328125,
    -4.25,
    -4.171875,
    -2.595703125,
    -0.8662109375,
    -3.169921875,
    -2.646484375,
    -1.248046875,
    -1.7822265625,
    -1.728515625,
    -2.810546875,
    -0.61767578125,
    -1.59765625,
    -1.9599609375,
    -1.34765625,
    -1.0947265625,
    -2.759765625,
    -1.7451171875,
    -2.44921875,
    -2.146484375,
    -3.603515625,
    -1.9912109375,
    -1.9814453125,
    -2.298828125,
    -2.072265625,
    -2.8203125,
    -0.62646484375,
    -1.755859375,
    -3.533203125,
    -3.0703125,
    -2.646484375,
    -2.126953125,
    -3.275390625,
    -2.201171875,
    -2.63671875,
    -1.892578125,
    -2.33984375,
    -1.77734375,
    -1.4267578125,
    -1.400390625,
    -2.029296875,
    -2.197265625,
    -0.41650390625,
    -1.6376953125,
    -2.521484375,
    -1.248046875,
    -1.392578125,
    -1.6689453125,
    -1.9375,
    -1.48046875,
    -1.845703125,
    -0.865234375,
    -0.93212890625,
    -1.7587890625,
    -1.40234375,
    -1.6337890625,
    -2.076171875,
    -0.8369140625,
    -2.169921875,
    -2.3046875,
    0.0188751220703125,
    -1.7021484375,
    -1.1650390625,
    -0.923828125,
    -0.023956298828125,
    -0.5400390625,
    -2.568359375,
    -0.85107421875,
    -1.5166015625,
    -1.8583984375,
    -1.1416015625,
    -1.5966796875,
    -0.8896484375,
    -1.560546875,
    -0.89453125,
    -1.021484375,
    -0.1458740234375,
    -0.1044921875,
    -0.58642578125,
    1.0078125,
    0.86083984375,
    -0.1531982421875,
    -0.2568359375,
    -0.333984375,
    0.26123046875,
    -0.96875,
    0.4453125,
    0.7294921875,
    1.4111328125,
    -0.416015625,
    -0.0232696533203125,
    -0.802734375,
    0.35986328125,
    0.69189453125,
    -0.79248046875,
    -0.1651611328125,
    -0.383056640625,
    0.71044921875,
    0.42724609375,
    -0.70263671875,
    -0.59521484375,
    2.0703125,
    -0.220703125,
    -0.302978515625,
    -0.393310546875,
    0.344970703125,
    -0.48583984375,
    -5.84765625
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9834395051002502,
    0.016184968873858452,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003754949430003762
   ],
   "p": [
    0.9838089346885681,
    0.01619104854762554
   ],
   "p_oracle": [
    0.9837594032287598,
    0.01624060794711113
   ],
   "abstain": 0.0003754949430003762,
   "abstain_oracle": 0.00037471865653060377,
   "abs_delta_abstain": 7.762864697724581e-07,
   "oracle_margin": 0.9675188064575195,
   "max_abs_delta_p": 4.9559399485588074e-05,
   "mean_abs_delta_p": 4.954542964696884e-05,
   "max_abs_delta_raw_logits": 0.006044864654541016,
   "mean_abs_delta_raw_logits": 0.0011555297242011875,
   "logits_min": -7.76171875,
   "logits_max": 2.076171875,
   "raw_logits_min": -7.76171875,
   "raw_logits_max": 2.076171875,
   "raw_logits_absmax": 7.76171875,
   "oracle_logits_min": -7.764531135559082,
   "oracle_logits_max": 2.0756096839904785,
   "oracle_raw_logits_absmax": 7.764531135559082,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3403149589430541,
   "seconds": 0.3403149589430541,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r14-fill",
   "request_id": "r14",
   "question_id": "fill",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 87,
   "nopts": 3,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    2.583984375,
    1.794921875,
    -2.443359375,
    -2.859375,
    -2.48828125,
    -3.189453125,
    -3.015625,
    -3.837890625,
    -4.06640625,
    -2.71875,
    -2.689453125,
    -3.779296875,
    -4.5,
    -2.865234375,
    -3.572265625,
    -4.08984375,
    -3.826171875,
    -4.83984375,
    -2.732421875,
    -3.369140625,
    -3.189453125,
    -1.625,
    -4.9296875,
    -6.12890625,
    -3.185546875,
    -4.5078125,
    -2.1796875,
    -4.9296875,
    -4.04296875,
    -6.5,
    -3.9296875,
    -4.53125,
    -5.6484375,
    -4.6171875,
    -3.056640625,
    -4.921875,
    -5.0,
    -5.671875,
    -4.15234375,
    -2.951171875,
    -6.703125,
    -4.4453125,
    -4.94921875,
    -5.51953125,
    -3.76953125,
    -4.67578125,
    -3.8125,
    -4.42578125,
    -4.96484375,
    -4.40625,
    -5.359375,
    -4.3828125,
    -3.0,
    -4.8359375,
    -3.0234375,
    -3.44921875,
    -3.8359375,
    -4.37890625,
    -4.359375,
    -5.8125,
    -7.265625,
    -6.50390625,
    -7.87890625,
    -5.72265625,
    -3.6484375,
    -6.984375,
    -7.2734375,
    -2.201171875,
    -7.3515625,
    -4.0625,
    -7.6484375,
    -4.796875,
    -2.78125,
    -5.4296875,
    -3.66796875,
    -5.82421875,
    -4.10546875,
    -6.5703125,
    -3.611328125,
    -5.43359375,
    -4.0859375,
    -3.693359375,
    -2.53515625,
    -4.24609375,
    -3.259765625,
    -3.8984375,
    -2.369140625,
    -3.68359375,
    -2.921875,
    -4.53125,
    -3.296875,
    -3.94140625,
    -3.734375,
    -3.6640625,
    -4.0625,
    -4.0546875,
    -4.7890625,
    -3.544921875,
    -2.998046875,
    -4.0,
    -5.4375,
    -4.75,
    -3.169921875,
    -3.900390625,
    -4.19921875,
    -5.7890625,
    -3.791015625,
    -5.17578125,
    -3.935546875,
    -5.58203125,
    -4.265625,
    -4.6953125,
    -4.76953125,
    -6.1171875,
    -4.734375,
    -3.775390625,
    -4.62109375,
    -5.48046875,
    -3.88671875,
    -3.4765625,
    -3.353515625,
    -4.0390625,
    -3.1328125,
    -3.53515625,
    -2.4140625,
    -5.19921875,
    -5.41796875,
    -2.6796875,
    -2.640625,
    -5.1171875,
    -5.40234375,
    -3.03125,
    -5.27734375,
    -5.84765625,
    -4.95703125,
    -4.15625,
    -3.529296875,
    -4.8828125,
    -5.6328125,
    -2.8515625,
    -3.53515625,
    -4.09375,
    -3.15625,
    -2.68359375,
    -4.80078125,
    -5.12109375,
    -5.77734375,
    -4.9921875,
    -4.7421875,
    -4.21875,
    -4.33203125,
    -2.7265625,
    -1.0126953125,
    -3.080078125,
    -2.599609375,
    -1.361328125,
    -1.7841796875,
    -1.912109375,
    -2.982421875,
    -0.72412109375,
    -1.830078125,
    -2.0,
    -1.544921875,
    -1.1259765625,
    -2.576171875,
    -1.939453125,
    -2.666015625,
    -2.125,
    -3.458984375,
    -2.033203125,
    -2.10546875,
    -2.486328125,
    -2.166015625,
    -2.9296875,
    -0.7900390625,
    -2.109375,
    -3.576171875,
    -3.375,
    -2.90625,
    -2.583984375,
    -3.224609375,
    -2.6484375,
    -2.486328125,
    -2.15234375,
    -2.197265625,
    -1.728515625,
    -1.37890625,
    -1.4892578125,
    -2.3671875,
    -2.26171875,
    -0.56298828125,
    -1.640625,
    -2.607421875,
    -1.3642578125,
    -1.4501953125,
    -1.5283203125,
    -2.06640625,
    -1.595703125,
    -1.8466796875,
    -1.009765625,
    -0.96484375,
    -1.7490234375,
    -1.3896484375,
    -1.5595703125,
    -2.26171875,
    -0.94482421875,
    -2.46875,
    -2.640625,
    -0.37646484375,
    -2.248046875,
    -1.154296875,
    -1.24609375,
    -0.206787109375,
    -0.99560546875,
    -2.4765625,
    -1.0693359375,
    -1.62109375,
    -2.16015625,
    -1.2255859375,
    -1.955078125,
    -1.3544921875,
    -1.7939453125,
    -1.0849609375,
    -0.91162109375,
    -0.2476806640625,
    0.09613037109375,
    -0.53173828125,
    1.09375,
    0.8486328125,
    -0.29345703125,
    -0.477783203125,
    -0.744140625,
    0.0833740234375,
    -0.89599609375,
    0.11138916015625,
    0.908203125,
    1.3984375,
    -0.313232421875,
    0.23828125,
    -0.8427734375,
    0.361083984375,
    0.58251953125,
    -0.5439453125,
    -0.1513671875,
    -0.09283447265625,
    0.467041015625,
    0.4833984375,
    -0.242431640625,
    -0.8935546875,
    2.02734375,
    -0.3603515625,
    -0.449462890625,
    -0.2080078125,
    0.328369140625,
    -0.57421875,
    -5.29296875
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.6766192317008972,
    0.3175639510154724,
    0.005461263936012983,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00035555375507101417
   ],
   "p": [
    0.6768598556518555,
    0.3176769018173218,
    0.005463206209242344
   ],
   "p_oracle": [
    0.675521969795227,
    0.3190520703792572,
    0.005425968673080206
   ],
   "abstain": 0.00035555375507101417,
   "abstain_oracle": 0.000354081392288208,
   "abs_delta_abstain": 1.472362782806158e-06,
   "oracle_margin": 0.35646989941596985,
   "max_abs_delta_p": 0.0013751685619354248,
   "mean_abs_delta_p": 0.0009167639849086603,
   "max_abs_delta_raw_logits": 0.008677959442138672,
   "mean_abs_delta_raw_logits": 0.0017450845334678888,
   "logits_min": -7.87890625,
   "logits_max": 2.583984375,
   "raw_logits_min": -7.87890625,
   "raw_logits_max": 2.583984375,
   "raw_logits_absmax": 7.87890625,
   "oracle_logits_min": -7.882862567901611,
   "oracle_logits_max": 2.586092710494995,
   "oracle_raw_logits_absmax": 7.882862567901611,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4859221250517294,
   "seconds": 0.4859221250517294,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r15-color",
   "request_id": "r15",
   "question_id": "color",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 53,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.765625,
    3.365234375,
    -2.759765625,
    -2.9453125,
    -2.298828125,
    -3.171875,
    -3.10546875,
    -3.79296875,
    -3.931640625,
    -2.775390625,
    -3.203125,
    -3.921875,
    -5.4921875,
    -3.720703125,
    -4.4453125,
    -4.6171875,
    -3.90234375,
    -4.16015625,
    -3.173828125,
    -3.248046875,
    -3.63671875,
    -2.267578125,
    -5.47265625,
    -7.25390625,
    -4.18359375,
    -5.78515625,
    -2.74609375,
    -5.5390625,
    -4.63671875,
    -6.89453125,
    -4.76953125,
    -4.8046875,
    -6.41796875,
    -5.21875,
    -3.341796875,
    -5.703125,
    -5.48046875,
    -5.92578125,
    -4.45703125,
    -3.169921875,
    -7.4375,
    -4.69140625,
    -4.875,
    -6.28125,
    -4.609375,
    -5.36328125,
    -4.546875,
    -4.55859375,
    -5.34765625,
    -4.921875,
    -5.8515625,
    -4.73046875,
    -3.02734375,
    -5.0625,
    -2.91796875,
    -3.3984375,
    -3.42578125,
    -4.4921875,
    -4.79296875,
    -5.6328125,
    -7.73046875,
    -6.60546875,
    -7.78515625,
    -6.796875,
    -3.775390625,
    -7.73828125,
    -7.86328125,
    -2.3671875,
    -8.109375,
    -4.296875,
    -8.171875,
    -5.6171875,
    -2.65234375,
    -6.32421875,
    -3.8671875,
    -6.515625,
    -4.4296875,
    -7.1015625,
    -4.3671875,
    -5.890625,
    -4.2734375,
    -4.34375,
    -2.8203125,
    -4.46484375,
    -3.810546875,
    -4.16796875,
    -2.705078125,
    -4.15625,
    -3.0625,
    -5.25390625,
    -3.681640625,
    -4.1953125,
    -4.421875,
    -4.41015625,
    -4.7421875,
    -4.921875,
    -5.24609375,
    -3.923828125,
    -4.27734375,
    -4.234375,
    -6.45703125,
    -4.671875,
    -3.7734375,
    -4.390625,
    -4.6796875,
    -6.37890625,
    -4.51171875,
    -5.9296875,
    -4.328125,
    -5.86328125,
    -4.5859375,
    -4.94921875,
    -5.42578125,
    -6.71875,
    -4.984375,
    -4.1640625,
    -4.71875,
    -6.33203125,
    -4.60546875,
    -3.73046875,
    -3.498046875,
    -4.00390625,
    -3.404296875,
    -3.919921875,
    -2.576171875,
    -6.015625,
    -5.98828125,
    -3.333984375,
    -3.32421875,
    -5.69921875,
    -5.890625,
    -3.89453125,
    -5.6953125,
    -6.15625,
    -5.18359375,
    -5.0234375,
    -4.23046875,
    -5.25390625,
    -6.0625,
    -3.1953125,
    -3.826171875,
    -4.3046875,
    -3.87890625,
    -3.490234375,
    -4.83984375,
    -5.53125,
    -6.3828125,
    -5.2890625,
    -4.80859375,
    -4.6484375,
    -4.921875,
    -3.294921875,
    -1.2841796875,
    -3.59375,
    -2.427734375,
    -1.810546875,
    -2.04296875,
    -2.083984375,
    -3.4765625,
    -0.475341796875,
    -2.185546875,
    -2.119140625,
    -1.720703125,
    -1.3134765625,
    -2.634765625,
    -2.3984375,
    -2.96875,
    -1.8212890625,
    -3.552734375,
    -2.1875,
    -2.609375,
    -2.482421875,
    -2.580078125,
    -3.169921875,
    -0.9345703125,
    -2.083984375,
    -3.3203125,
    -3.36328125,
    -3.2109375,
    -3.751953125,
    -3.57421875,
    -3.427734375,
    -2.8359375,
    -2.326171875,
    -2.75390625,
    -1.79296875,
    -1.3095703125,
    -1.9677734375,
    -2.189453125,
    -2.265625,
    -1.2216796875,
    -1.8828125,
    -3.189453125,
    -1.8720703125,
    -1.6015625,
    -2.302734375,
    -2.099609375,
    -2.1171875,
    -1.791015625,
    -1.6806640625,
    -1.4482421875,
    -2.10546875,
    -1.765625,
    -2.048828125,
    -2.537109375,
    -1.337890625,
    -2.794921875,
    -2.734375,
    -0.420654296875,
    -2.435546875,
    -1.02734375,
    -1.390625,
    -0.994140625,
    -1.73046875,
    -2.53515625,
    -1.2705078125,
    -1.796875,
    -2.509765625,
    -2.12109375,
    -2.17578125,
    -1.533203125,
    -1.6748046875,
    -1.505859375,
    -1.30078125,
    -0.2078857421875,
    0.0645751953125,
    -0.40576171875,
    0.814453125,
    1.134765625,
    -0.059417724609375,
    -0.78564453125,
    -0.5322265625,
    0.1673583984375,
    -0.76318359375,
    0.49658203125,
    1.158203125,
    1.3232421875,
    -0.6982421875,
    -0.0556640625,
    -0.7548828125,
    0.2041015625,
    0.82763671875,
    -0.95068359375,
    -0.11077880859375,
    0.20361328125,
    0.56298828125,
    0.385986328125,
    -0.2451171875,
    -0.96728515625,
    1.3486328125,
    -0.046844482421875,
    -0.73291015625,
    -0.30615234375,
    0.4345703125,
    -0.6298828125,
    -3.921875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.0030307031702250242,
    0.9929051995277405,
    0.0030475249513983727,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.001016618451103568
   ],
   "p": [
    0.0030337872449308634,
    0.9939155578613281,
    0.0030506260227411985
   ],
   "p_oracle": [
    0.0030322920065373182,
    0.9939413070678711,
    0.0030264435335993767
   ],
   "abstain": 0.001016618451103568,
   "abstain_oracle": 0.001004166784696281,
   "abs_delta_abstain": 1.245166640728712e-05,
   "oracle_margin": 0.9909090399742126,
   "max_abs_delta_p": 2.574920654296875e-05,
   "mean_abs_delta_p": 1.7142311359445255e-05,
   "max_abs_delta_raw_logits": 0.009699583053588867,
   "mean_abs_delta_raw_logits": 0.002699704229598865,
   "logits_min": -8.171875,
   "logits_max": 3.365234375,
   "raw_logits_min": -8.171875,
   "raw_logits_max": 3.365234375,
   "raw_logits_absmax": 8.171875,
   "oracle_logits_min": -8.178394317626953,
   "oracle_logits_max": 3.368619918823242,
   "oracle_raw_logits_absmax": 8.178394317626953,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.2997432919219136,
   "seconds": 0.2997432919219136,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r15-level",
   "request_id": "r15",
   "question_id": "level",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 125,
   "nopts": 10,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    -1.62890625,
    -2.20703125,
    -2.525390625,
    -2.552734375,
    -2.353515625,
    -3.453125,
    -2.919921875,
    4.89453125,
    -2.33203125,
    -0.62890625,
    -2.91796875,
    -2.83203125,
    -4.6796875,
    -3.287109375,
    -3.859375,
    -3.9375,
    -3.517578125,
    -4.76171875,
    -2.990234375,
    -2.611328125,
    -3.69140625,
    -1.7314453125,
    -4.13671875,
    -4.36328125,
    -3.35546875,
    -4.31640625,
    -3.181640625,
    -5.1484375,
    -4.72265625,
    -5.99609375,
    -4.37890625,
    -5.0546875,
    -6.0234375,
    -4.8671875,
    -3.0,
    -4.4921875,
    -5.30859375,
    -5.7421875,
    -4.40234375,
    -2.60546875,
    -6.47265625,
    -3.685546875,
    -4.7890625,
    -6.390625,
    -4.05859375,
    -4.0546875,
    -4.2109375,
    -5.171875,
    -4.7421875,
    -4.234375,
    -5.50390625,
    -4.02734375,
    -2.798828125,
    -5.09765625,
    -3.42578125,
    -3.3125,
    -4.22265625,
    -4.76953125,
    -4.390625,
    -5.41015625,
    -7.44140625,
    -6.9453125,
    -7.81640625,
    -5.69140625,
    -3.74609375,
    -7.3984375,
    -6.7890625,
    -1.9248046875,
    -7.35546875,
    -4.20703125,
    -7.75,
    -5.5234375,
    -2.486328125,
    -5.6796875,
    -3.421875,
    -6.37109375,
    -4.39453125,
    -6.72265625,
    -3.751953125,
    -5.61328125,
    -4.515625,
    -4.34375,
    -2.404296875,
    -4.53125,
    -3.080078125,
    -4.8515625,
    -2.2890625,
    -3.884765625,
    -2.61328125,
    -3.9609375,
    -3.416015625,
    -3.5078125,
    -4.08984375,
    -3.3984375,
    -4.859375,
    -4.29296875,
    -5.34765625,
    -3.216796875,
    -3.220703125,
    -4.49609375,
    -5.73046875,
    -4.484375,
    -3.859375,
    -3.861328125,
    -4.54296875,
    -5.62890625,
    -3.61328125,
    -5.265625,
    -4.07421875,
    -5.7265625,
    -4.421875,
    -4.7109375,
    -4.5546875,
    -5.609375,
    -5.078125,
    -3.95703125,
    -4.55859375,
    -6.61328125,
    -4.16015625,
    -3.23828125,
    -3.17578125,
    -3.720703125,
    -3.220703125,
    -3.390625,
    -2.09765625,
    -4.95703125,
    -5.21875,
    -2.505859375,
    -2.9375,
    -5.4609375,
    -5.8515625,
    -2.458984375,
    -5.35546875,
    -5.6953125,
    -4.9609375,
    -4.6015625,
    -3.54296875,
    -4.8125,
    -5.90234375,
    -2.26953125,
    -2.94140625,
    -3.677734375,
    -3.51953125,
    -2.521484375,
    -4.640625,
    -4.86328125,
    -5.9921875,
    -5.08203125,
    -5.54296875,
    -5.421875,
    -4.13671875,
    -2.96875,
    -0.91015625,
    -3.609375,
    -2.466796875,
    -1.40234375,
    -1.5048828125,
    -2.005859375,
    -3.232421875,
    -0.058197021484375,
    -1.3662109375,
    -2.197265625,
    -1.7275390625,
    -1.4169921875,
    -3.21484375,
    -1.234375,
    -3.345703125,
    -2.50390625,
    -3.61328125,
    -2.505859375,
    -1.8525390625,
    -2.58203125,
    -2.001953125,
    -3.15625,
    -0.198974609375,
    -1.751953125,
    -3.662109375,
    -3.224609375,
    -3.263671875,
    -2.267578125,
    -3.666015625,
    -1.93359375,
    -2.6171875,
    -1.6025390625,
    -2.39453125,
    -1.6943359375,
    -0.96923828125,
    -2.09375,
    -1.8408203125,
    -2.05859375,
    -1.1279296875,
    -1.5390625,
    -2.484375,
    -1.8212890625,
    -1.72265625,
    -1.6328125,
    -1.349609375,
    -1.8310546875,
    -1.83203125,
    -1.013671875,
    -1.30859375,
    -2.169921875,
    -1.5791015625,
    -1.88671875,
    -2.240234375,
    -0.9443359375,
    -2.705078125,
    -2.57421875,
    0.236328125,
    -2.3828125,
    -1.4140625,
    -1.7783203125,
    -0.2235107421875,
    -1.2900390625,
    -2.328125,
    -1.119140625,
    -1.5498046875,
    -1.9658203125,
    -0.75537109375,
    -1.6806640625,
    -1.43359375,
    -1.1337890625,
    -1.0029296875,
    -1.037109375,
    -0.1527099609375,
    -0.406494140625,
    -0.477294921875,
    1.2197265625,
    1.1015625,
    -0.3916015625,
    -0.269775390625,
    -0.72216796875,
    0.421875,
    -0.28515625,
    0.266845703125,
    1.1083984375,
    1.255859375,
    -0.3427734375,
    -0.411376953125,
    -0.76123046875,
    -0.317626953125,
    0.8173828125,
    -0.83984375,
    -0.1903076171875,
    -0.05059814453125,
    0.72705078125,
    0.11505126953125,
    -0.2034912109375,
    -0.43212890625,
    1.9794921875,
    0.2022705078125,
    0.040008544921875,
    -0.70263671875,
    0.2205810546875,
    -0.724609375,
    -5.44921875
   ],
   "raw256_argmax": 7,
   "option_argmax": 7,
   "oracle_argmax": 7,
   "argmax_agrees": true,
   "p_full": [
    0.0018996113212779164,
    0.001091370009817183,
    0.0008043188136070967,
    0.000783508934546262,
    0.0009483867906965315,
    0.00033050740603357553,
    0.0005510256160050631,
    0.9876199960708618,
    0.0009681219235062599,
    0.004954480100423098,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    4.8768630222184584e-05
   ],
   "p": [
    0.0018997038714587688,
    0.001091423211619258,
    0.0008043579873628914,
    0.0007835471187718213,
    0.0009484329493716359,
    0.0003305235004518181,
    0.0005510524497367442,
    0.9876680970191956,
    0.0009681690717115998,
    0.004954721312969923
   ],
   "p_oracle": [
    0.0018778711091727018,
    0.001082923263311386,
    0.0007980354130268097,
    0.0007786157075315714,
    0.0009413186344318092,
    0.00032714440021663904,
    0.0005481148837134242,
    0.9876933097839355,
    0.0009699876536615193,
    0.004982673097401857
   ],
   "abstain": 4.8768630222184584e-05,
   "abstain_oracle": 4.8452511691721156e-05,
   "abs_delta_abstain": 3.161185304634273e-07,
   "oracle_margin": 0.9827106595039368,
   "max_abs_delta_p": 2.7951784431934357e-05,
   "mean_abs_delta_p": 1.1000080849044026e-05,
   "max_abs_delta_raw_logits": 0.01256561279296875,
   "mean_abs_delta_raw_logits": 0.0016537345363758504,
   "logits_min": -7.81640625,
   "logits_max": 4.89453125,
   "raw_logits_min": -7.81640625,
   "raw_logits_max": 4.89453125,
   "raw_logits_absmax": 7.81640625,
   "oracle_logits_min": -7.816807746887207,
   "oracle_logits_max": 4.9012556076049805,
   "oracle_raw_logits_absmax": 7.816807746887207,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.6955912910634652,
   "seconds": 0.6955912910634652,
   "full_vocab_argmax_id": 7,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r15-position",
   "request_id": "r15",
   "question_id": "position",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 59,
   "nopts": 2,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    2.3046875,
    -0.66064453125,
    -1.9599609375,
    -2.494140625,
    -1.7841796875,
    -2.685546875,
    -2.48828125,
    -3.30859375,
    -3.4921875,
    -1.7958984375,
    -2.16796875,
    -3.00390625,
    -3.830078125,
    -2.16796875,
    -3.20703125,
    -3.748046875,
    -3.861328125,
    -4.9921875,
    -2.751953125,
    -2.83984375,
    -3.15234375,
    -1.5341796875,
    -4.79296875,
    -5.74609375,
    -3.04296875,
    -3.7890625,
    -2.083984375,
    -4.19140625,
    -3.6171875,
    -5.74609375,
    -3.7265625,
    -3.798828125,
    -5.41015625,
    -3.931640625,
    -2.65625,
    -4.25390625,
    -4.78125,
    -5.765625,
    -3.95703125,
    -2.830078125,
    -6.31640625,
    -3.9921875,
    -4.6328125,
    -5.578125,
    -3.357421875,
    -4.21484375,
    -3.642578125,
    -4.42578125,
    -4.828125,
    -3.927734375,
    -5.11328125,
    -4.15234375,
    -2.853515625,
    -5.02734375,
    -2.794921875,
    -3.287109375,
    -3.80078125,
    -4.515625,
    -4.2421875,
    -5.81640625,
    -6.80859375,
    -6.265625,
    -7.6484375,
    -5.484375,
    -3.455078125,
    -6.796875,
    -6.96875,
    -2.318359375,
    -7.109375,
    -3.99609375,
    -7.3359375,
    -4.69921875,
    -2.60546875,
    -5.05859375,
    -3.47265625,
    -5.48828125,
    -3.80859375,
    -6.43359375,
    -3.4296875,
    -5.22265625,
    -4.125,
    -3.630859375,
    -2.390625,
    -4.11328125,
    -3.138671875,
    -3.994140625,
    -2.119140625,
    -3.62890625,
    -2.92578125,
    -4.10546875,
    -3.146484375,
    -3.853515625,
    -3.6171875,
    -3.359375,
    -4.125,
    -3.9765625,
    -4.53125,
    -3.19140625,
    -2.6953125,
    -3.888671875,
    -4.9140625,
    -4.44921875,
    -3.109375,
    -3.63671875,
    -4.11328125,
    -5.23046875,
    -3.244140625,
    -4.72265625,
    -3.744140625,
    -5.37890625,
    -4.02734375,
    -4.26171875,
    -4.34375,
    -5.45703125,
    -4.44921875,
    -3.455078125,
    -4.41015625,
    -5.1953125,
    -3.453125,
    -3.19140625,
    -2.962890625,
    -4.109375,
    -2.8828125,
    -3.376953125,
    -2.033203125,
    -4.77734375,
    -5.109375,
    -2.37109375,
    -2.453125,
    -4.8203125,
    -5.33984375,
    -2.419921875,
    -5.1015625,
    -5.65625,
    -4.84375,
    -3.673828125,
    -3.19140625,
    -4.390625,
    -5.4375,
    -2.392578125,
    -3.240234375,
    -3.849609375,
    -2.900390625,
    -2.458984375,
    -4.55078125,
    -4.65625,
    -5.1640625,
    -4.9765625,
    -4.84375,
    -4.0546875,
    -4.18359375,
    -2.365234375,
    -0.54736328125,
    -2.91796875,
    -2.369140625,
    -1.126953125,
    -1.4609375,
    -1.7900390625,
    -2.740234375,
    -0.448486328125,
    -1.6630859375,
    -1.8720703125,
    -1.279296875,
    -0.8876953125,
    -2.517578125,
    -1.74609375,
    -2.337890625,
    -2.0234375,
    -3.314453125,
    -1.771484375,
    -1.779296875,
    -2.25390625,
    -1.9033203125,
    -2.7265625,
    -0.364501953125,
    -1.56640625,
    -3.224609375,
    -3.052734375,
    -2.5,
    -1.9599609375,
    -2.931640625,
    -1.8818359375,
    -2.3515625,
    -1.859375,
    -2.060546875,
    -1.615234375,
    -1.25,
    -1.2939453125,
    -2.111328125,
    -2.013671875,
    -0.09051513671875,
    -1.5078125,
    -2.216796875,
    -1.2724609375,
    -1.2685546875,
    -1.400390625,
    -1.93359375,
    -1.34375,
    -1.708984375,
    -0.82373046875,
    -0.8291015625,
    -1.4560546875,
    -1.1630859375,
    -1.40234375,
    -2.044921875,
    -0.62060546875,
    -2.130859375,
    -2.13671875,
    -0.0850830078125,
    -1.7109375,
    -0.8896484375,
    -0.9326171875,
    0.18603515625,
    -0.4638671875,
    -2.478515625,
    -0.72998046875,
    -1.416015625,
    -1.8603515625,
    -0.998046875,
    -1.6787109375,
    -1.1669921875,
    -1.7470703125,
    -0.91748046875,
    -0.9013671875,
    -0.14306640625,
    0.08251953125,
    -0.498046875,
    1.162109375,
    0.83642578125,
    -0.1632080078125,
    -0.424560546875,
    -0.72021484375,
    0.326904296875,
    -0.8408203125,
    0.277099609375,
    0.72314453125,
    1.37109375,
    -0.1505126953125,
    -0.0760498046875,
    -0.82373046875,
    0.473388671875,
    0.7060546875,
    -0.58642578125,
    -0.1494140625,
    -0.1722412109375,
    0.5126953125,
    0.55517578125,
    -0.5498046875,
    -0.83642578125,
    2.13671875,
    -0.460205078125,
    -0.4326171875,
    -0.261474609375,
    0.33203125,
    -0.457275390625,
    -5.90234375
   ],
   "raw256_argmax": 0,
   "option_argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "p_full": [
    0.9445980787277222,
    0.05504022538661957,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003617306938394904
   ],
   "p": [
    0.9449398517608643,
    0.055060140788555145
   ],
   "p_oracle": [
    0.9450725317001343,
    0.05492749810218811
   ],
   "abstain": 0.0003617306938394904,
   "abstain_oracle": 0.00036147201899439096,
   "abs_delta_abstain": 2.5867484509944916e-07,
   "oracle_margin": 0.8901450634002686,
   "max_abs_delta_p": 0.00013267993927001953,
   "mean_abs_delta_p": 0.00013266131281852722,
   "max_abs_delta_raw_logits": 0.0033996105194091797,
   "mean_abs_delta_raw_logits": 0.000890132156200707,
   "logits_min": -7.6484375,
   "logits_max": 2.3046875,
   "raw_logits_min": -7.6484375,
   "raw_logits_max": 2.3046875,
   "raw_logits_absmax": 7.6484375,
   "oracle_logits_min": -7.648866653442383,
   "oracle_logits_max": 2.306751251220703,
   "oracle_raw_logits_absmax": 7.648866653442383,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3332182909362018,
   "seconds": 0.3332182909362018,
   "full_vocab_argmax_id": 0,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r16-color",
   "request_id": "r16",
   "question_id": "color",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 71,
   "nopts": 3,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -2.373046875,
    3.09765625,
    -2.828125,
    -3.130859375,
    -2.537109375,
    -2.947265625,
    -3.021484375,
    -4.01953125,
    -3.849609375,
    -2.880859375,
    -3.283203125,
    -4.0703125,
    -5.35546875,
    -3.484375,
    -4.3203125,
    -4.49609375,
    -3.712890625,
    -4.16015625,
    -3.1796875,
    -3.30078125,
    -3.533203125,
    -2.208984375,
    -5.34765625,
    -7.109375,
    -3.93359375,
    -5.44140625,
    -2.52734375,
    -5.31640625,
    -4.59765625,
    -6.68359375,
    -4.68359375,
    -4.66796875,
    -6.31640625,
    -5.1484375,
    -3.4765625,
    -5.60546875,
    -5.40625,
    -5.984375,
    -4.390625,
    -3.04296875,
    -7.171875,
    -4.5859375,
    -4.78125,
    -5.9765625,
    -4.58203125,
    -5.265625,
    -4.50390625,
    -4.4296875,
    -5.38671875,
    -4.6796875,
    -5.74609375,
    -4.79296875,
    -2.873046875,
    -4.91015625,
    -2.9375,
    -3.439453125,
    -3.578125,
    -4.3125,
    -4.69140625,
    -5.31640625,
    -7.4296875,
    -6.4140625,
    -7.57421875,
    -6.64453125,
    -3.775390625,
    -7.53515625,
    -7.765625,
    -2.419921875,
    -7.9375,
    -4.12109375,
    -7.8125,
    -5.38671875,
    -2.564453125,
    -6.30078125,
    -3.73828125,
    -6.31640625,
    -4.30859375,
    -6.95703125,
    -4.3828125,
    -5.78125,
    -4.39453125,
    -4.29296875,
    -2.767578125,
    -4.4609375,
    -3.677734375,
    -4.06640625,
    -2.623046875,
    -3.978515625,
    -3.154296875,
    -5.078125,
    -3.56640625,
    -4.1953125,
    -4.44140625,
    -4.24609375,
    -4.59375,
    -4.765625,
    -5.140625,
    -4.0390625,
    -4.0703125,
    -4.1484375,
    -6.2734375,
    -4.49609375,
    -3.736328125,
    -4.2578125,
    -4.48828125,
    -6.3046875,
    -4.43359375,
    -5.8046875,
    -4.14453125,
    -5.73828125,
    -4.46875,
    -4.83984375,
    -5.23828125,
    -6.44921875,
    -4.86328125,
    -4.0234375,
    -4.65234375,
    -6.265625,
    -4.59765625,
    -3.56640625,
    -3.34765625,
    -3.923828125,
    -3.279296875,
    -3.8203125,
    -2.333984375,
    -5.9765625,
    -5.9140625,
    -3.201171875,
    -3.236328125,
    -5.58203125,
    -5.66015625,
    -3.9296875,
    -5.70703125,
    -6.11328125,
    -4.80859375,
    -4.90234375,
    -3.970703125,
    -4.98046875,
    -5.99609375,
    -3.15625,
    -3.720703125,
    -4.27734375,
    -3.814453125,
    -3.5,
    -4.76953125,
    -5.359375,
    -6.234375,
    -5.13671875,
    -4.68359375,
    -4.41796875,
    -4.859375,
    -3.2578125,
    -1.330078125,
    -3.498046875,
    -2.328125,
    -1.755859375,
    -2.013671875,
    -2.00390625,
    -3.470703125,
    -0.51416015625,
    -2.15625,
    -1.9658203125,
    -1.8193359375,
    -1.3779296875,
    -2.4921875,
    -2.29296875,
    -2.87109375,
    -1.8818359375,
    -3.53515625,
    -2.15625,
    -2.453125,
    -2.525390625,
    -2.55078125,
    -3.23046875,
    -0.85498046875,
    -2.236328125,
    -3.333984375,
    -3.49609375,
    -3.318359375,
    -3.7109375,
    -3.763671875,
    -3.474609375,
    -2.693359375,
    -2.396484375,
    -2.728515625,
    -1.7939453125,
    -1.3994140625,
    -1.8564453125,
    -2.244140625,
    -2.365234375,
    -1.1708984375,
    -1.9873046875,
    -3.328125,
    -1.890625,
    -1.4990234375,
    -2.26953125,
    -2.16015625,
    -2.18359375,
    -1.7783203125,
    -1.71875,
    -1.482421875,
    -2.130859375,
    -1.9248046875,
    -2.1328125,
    -2.47265625,
    -1.326171875,
    -2.671875,
    -2.7421875,
    -0.471435546875,
    -2.408203125,
    -1.1025390625,
    -1.6064453125,
    -1.0107421875,
    -1.6484375,
    -2.658203125,
    -1.416015625,
    -2.068359375,
    -2.4921875,
    -2.029296875,
    -2.123046875,
    -1.56640625,
    -1.6962890625,
    -1.455078125,
    -1.32421875,
    -0.0548095703125,
    -0.00861358642578125,
    -0.48291015625,
    0.775390625,
    1.0322265625,
    -0.0299835205078125,
    -0.7392578125,
    -0.477294921875,
    0.09259033203125,
    -0.61767578125,
    0.444580078125,
    1.2041015625,
    1.2578125,
    -0.61865234375,
    0.001590728759765625,
    -0.70654296875,
    0.34033203125,
    0.833984375,
    -0.796875,
    -0.2047119140625,
    0.11346435546875,
    0.525390625,
    0.55859375,
    -0.2052001953125,
    -0.9521484375,
    1.48046875,
    -0.2191162109375,
    -0.6240234375,
    -0.337890625,
    0.39208984375,
    -0.439208984375,
    -3.189453125
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.00562715670093894,
    0.988109827041626,
    0.0036608316004276276,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0026021618396043777
   ],
   "p": [
    0.005641837604343891,
    0.9906877279281616,
    0.003670382546260953
   ],
   "p_oracle": [
    0.005624168086796999,
    0.9907371401786804,
    0.0036387406289577484
   ],
   "abstain": 0.0026021618396043777,
   "abstain_oracle": 0.0025959154590964317,
   "abs_delta_abstain": 6.246380507946014e-06,
   "oracle_margin": 0.9851129651069641,
   "max_abs_delta_p": 4.941225051879883e-05,
   "mean_abs_delta_p": 3.290789512296518e-05,
   "max_abs_delta_raw_logits": 0.007920265197753906,
   "mean_abs_delta_raw_logits": 0.002445367048494518,
   "logits_min": -7.9375,
   "logits_max": 3.09765625,
   "raw_logits_min": -7.9375,
   "raw_logits_max": 3.09765625,
   "raw_logits_absmax": 7.9375,
   "oracle_logits_min": -7.9451446533203125,
   "oracle_logits_max": 3.1033008098602295,
   "oracle_raw_logits_absmax": 7.9451446533203125,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.3971496250014752,
   "seconds": 0.3971496250014752,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r16-level",
   "request_id": "r16",
   "question_id": "level",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 160,
   "nopts": 10,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    -1.201171875,
    -2.4453125,
    -2.115234375,
    3.349609375,
    -1.447265625,
    -2.88671875,
    -3.322265625,
    -3.85546875,
    -4.43359375,
    -1.4853515625,
    -1.50390625,
    -2.98046875,
    -3.330078125,
    -1.5126953125,
    -3.52734375,
    -4.52734375,
    -4.375,
    -5.046875,
    -2.671875,
    -2.74609375,
    -3.74609375,
    -1.6708984375,
    -5.6953125,
    -5.37109375,
    -3.931640625,
    -4.3125,
    -2.759765625,
    -4.81640625,
    -3.5078125,
    -4.78125,
    -4.234375,
    -4.234375,
    -6.25,
    -2.9453125,
    -1.96875,
    -4.7734375,
    -4.921875,
    -5.671875,
    -3.876953125,
    -3.15234375,
    -6.703125,
    -4.1015625,
    -5.046875,
    -5.859375,
    -3.484375,
    -4.234375,
    -4.4375,
    -4.8671875,
    -5.42578125,
    -3.9453125,
    -4.47265625,
    -4.05078125,
    -2.8515625,
    -5.6484375,
    -2.337890625,
    -2.5234375,
    -4.3203125,
    -4.734375,
    -4.375,
    -5.82421875,
    -7.08984375,
    -6.65625,
    -7.73046875,
    -5.6484375,
    -3.47265625,
    -6.8046875,
    -7.3671875,
    -1.75390625,
    -7.40234375,
    -4.46875,
    -7.65625,
    -5.15625,
    -2.41015625,
    -5.10546875,
    -3.693359375,
    -5.6328125,
    -4.421875,
    -6.5546875,
    -3.50390625,
    -5.2265625,
    -4.20703125,
    -3.599609375,
    -2.34765625,
    -3.9609375,
    -3.087890625,
    -4.125,
    -2.5625,
    -2.986328125,
    -3.2734375,
    -4.296875,
    -3.642578125,
    -4.12109375,
    -3.837890625,
    -3.49609375,
    -4.54296875,
    -4.609375,
    -4.7734375,
    -3.34375,
    -3.390625,
    -3.72265625,
    -5.06640625,
    -5.1953125,
    -3.041015625,
    -3.685546875,
    -4.44921875,
    -5.3046875,
    -3.478515625,
    -4.41015625,
    -4.38671875,
    -5.51171875,
    -4.26171875,
    -4.73828125,
    -4.296875,
    -5.54296875,
    -4.875,
    -3.927734375,
    -4.04296875,
    -5.46875,
    -3.720703125,
    -2.830078125,
    -2.689453125,
    -4.1953125,
    -3.15234375,
    -3.724609375,
    -1.982421875,
    -5.14453125,
    -5.16796875,
    -2.654296875,
    -3.46875,
    -5.4453125,
    -5.31640625,
    -2.345703125,
    -5.8125,
    -5.9375,
    -4.38671875,
    -4.26171875,
    -3.783203125,
    -4.73046875,
    -5.7265625,
    -2.287109375,
    -3.044921875,
    -4.2578125,
    -3.12109375,
    -3.060546875,
    -4.05078125,
    -4.625,
    -5.74609375,
    -5.5234375,
    -5.609375,
    -4.48046875,
    -4.21484375,
    -2.611328125,
    -1.2685546875,
    -3.572265625,
    -2.42578125,
    -1.482421875,
    -1.7763671875,
    -1.6865234375,
    -2.673828125,
    -1.1337890625,
    -2.279296875,
    -2.6328125,
    -1.5576171875,
    -1.0107421875,
    -3.404296875,
    -1.6904296875,
    -2.5546875,
    -2.26171875,
    -3.947265625,
    -2.056640625,
    -1.828125,
    -2.173828125,
    -2.171875,
    -3.240234375,
    -1.060546875,
    -1.8076171875,
    -3.361328125,
    -3.044921875,
    -2.37890625,
    -2.806640625,
    -3.125,
    -2.31640625,
    -2.66015625,
    -1.724609375,
    -2.83203125,
    -1.953125,
    -1.552734375,
    -2.0546875,
    -2.005859375,
    -1.693359375,
    -0.9833984375,
    -1.39453125,
    -2.431640625,
    -1.4912109375,
    -1.4228515625,
    -2.15625,
    -1.5048828125,
    -1.291015625,
    -1.943359375,
    -1.00390625,
    -1.6083984375,
    -1.578125,
    -1.2373046875,
    -2.009765625,
    -2.037109375,
    -1.2001953125,
    -1.904296875,
    -2.736328125,
    0.256591796875,
    -2.11328125,
    -0.73046875,
    -1.30859375,
    -0.53515625,
    -1.078125,
    -2.333984375,
    -0.984375,
    -1.6787109375,
    -2.0546875,
    -1.310546875,
    -1.6865234375,
    -1.3935546875,
    -1.61328125,
    -1.0751953125,
    -1.4921875,
    -0.07470703125,
    0.2012939453125,
    -0.83349609375,
    0.798828125,
    1.0595703125,
    -0.4736328125,
    -0.9375,
    -0.51318359375,
    -0.45654296875,
    -1.095703125,
    0.55810546875,
    0.6806640625,
    1.2607421875,
    -0.5361328125,
    -0.4814453125,
    -0.485107421875,
    0.7041015625,
    0.4189453125,
    -0.76904296875,
    0.2822265625,
    -0.06866455078125,
    0.10101318359375,
    0.2841796875,
    -0.46240234375,
    -0.1746826171875,
    1.796875,
    -0.11553955078125,
    -0.62744140625,
    -0.57275390625,
    1.099609375,
    -0.167724609375,
    -5.34375
   ],
   "raw256_argmax": 3,
   "option_argmax": 3,
   "oracle_argmax": 3,
   "argmax_agrees": true,
   "p_full": [
    0.012165131978690624,
    0.003690962679684162,
    0.00506479712203145,
    0.9544634819030762,
    0.00960859190672636,
    0.002417504321783781,
    0.0015923356404528022,
    0.0009550903341732919,
    0.0005487211747094989,
    0.009264104999601841,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.00022930999693926424
   ],
   "p": [
    0.012167921289801598,
    0.003691809019073844,
    0.005065958481281996,
    0.9546823501586914,
    0.0096107954159379,
    0.002418058691546321,
    0.0015927007189020514,
    0.0009553093113936484,
    0.0005488470196723938,
    0.009266229346394539
   ],
   "p_oracle": [
    0.011903850361704826,
    0.0036331256851553917,
    0.0049773394130170345,
    0.9555612802505493,
    0.009435553103685379,
    0.0023825846146792173,
    0.00155331299174577,
    0.0009360660333186388,
    0.0005382035742513835,
    0.009078679606318474
   ],
   "abstain": 0.00022930999693926424,
   "abstain_oracle": 0.00022859007003717124,
   "abs_delta_abstain": 7.199269020929933e-07,
   "oracle_margin": 0.9436574578285217,
   "max_abs_delta_p": 0.0008789300918579102,
   "mean_abs_delta_p": 0.00017578440019860864,
   "max_abs_delta_raw_logits": 0.016265869140625,
   "mean_abs_delta_raw_logits": 0.002548632270190865,
   "logits_min": -7.73046875,
   "logits_max": 3.349609375,
   "raw_logits_min": -7.73046875,
   "raw_logits_max": 3.349609375,
   "raw_logits_absmax": 7.73046875,
   "oracle_logits_min": -7.726421356201172,
   "oracle_logits_max": 3.365875244140625,
   "oracle_raw_logits_absmax": 7.726421356201172,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.9172215000726283,
   "seconds": 0.9172215000726283,
   "full_vocab_argmax_id": 3,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r16-position",
   "request_id": "r16",
   "question_id": "position",
   "type": "score",
   "qtype_index": 1,
   "kind": "slot",
   "tokens": 81,
   "nopts": 2,
   "temperature": 1.0431410074234009,
   "raw_logits": [
    -2.4609375,
    1.8935546875,
    -1.2958984375,
    -2.119140625,
    -1.7978515625,
    -2.734375,
    -2.396484375,
    -3.05859375,
    -3.26171875,
    -1.60546875,
    -2.1640625,
    -3.333984375,
    -4.0,
    -2.087890625,
    -3.22265625,
    -4.0625,
    -3.78125,
    -4.4453125,
    -2.673828125,
    -2.431640625,
    -3.314453125,
    -1.55859375,
    -4.74609375,
    -6.1796875,
    -3.072265625,
    -4.43359375,
    -2.05859375,
    -4.48046875,
    -3.916015625,
    -5.80078125,
    -4.33203125,
    -3.56640625,
    -5.73828125,
    -4.31640625,
    -2.64453125,
    -4.859375,
    -4.96875,
    -5.796875,
    -4.0703125,
    -2.626953125,
    -6.69140625,
    -4.0390625,
    -4.37890625,
    -5.72265625,
    -3.583984375,
    -4.46875,
    -4.03125,
    -4.19140625,
    -5.0,
    -4.1328125,
    -5.390625,
    -4.55078125,
    -2.328125,
    -4.671875,
    -2.4921875,
    -3.359375,
    -3.4921875,
    -4.4375,
    -4.60546875,
    -5.2265625,
    -7.0078125,
    -5.96484375,
    -7.421875,
    -6.25390625,
    -3.0546875,
    -6.91015625,
    -7.0,
    -2.392578125,
    -7.30859375,
    -4.12890625,
    -7.42578125,
    -4.765625,
    -2.28515625,
    -5.3671875,
    -3.490234375,
    -5.49609375,
    -3.744140625,
    -6.21484375,
    -3.708984375,
    -5.05859375,
    -4.17578125,
    -3.416015625,
    -2.17578125,
    -3.966796875,
    -2.97265625,
    -3.728515625,
    -2.19140625,
    -3.51171875,
    -2.818359375,
    -4.03125,
    -2.986328125,
    -3.76953125,
    -3.708984375,
    -3.55859375,
    -4.26953125,
    -4.32421875,
    -4.65625,
    -3.32421875,
    -3.29296875,
    -3.78125,
    -5.0390625,
    -4.18359375,
    -3.318359375,
    -3.576171875,
    -4.0390625,
    -5.44140625,
    -3.4765625,
    -4.953125,
    -3.611328125,
    -5.0703125,
    -4.0703125,
    -4.13671875,
    -4.50390625,
    -5.59375,
    -4.26171875,
    -3.451171875,
    -4.23828125,
    -5.29296875,
    -3.9140625,
    -2.8828125,
    -3.03515625,
    -3.783203125,
    -2.951171875,
    -3.52734375,
    -2.2265625,
    -5.29296875,
    -4.84375,
    -2.759765625,
    -2.6875,
    -4.84375,
    -5.17578125,
    -2.716796875,
    -5.140625,
    -5.59765625,
    -4.81640625,
    -3.94921875,
    -3.388671875,
    -4.39453125,
    -5.41796875,
    -2.53515625,
    -3.0546875,
    -3.732421875,
    -3.21484375,
    -3.048828125,
    -4.4140625,
    -4.734375,
    -5.33203125,
    -4.625,
    -4.56640625,
    -4.04296875,
    -4.3671875,
    -2.765625,
    -0.6044921875,
    -3.091796875,
    -2.08984375,
    -1.05859375,
    -1.45703125,
    -1.9609375,
    -3.17578125,
    -0.2802734375,
    -2.076171875,
    -2.1015625,
    -1.228515625,
    -1.072265625,
    -2.5859375,
    -2.04296875,
    -2.451171875,
    -1.4365234375,
    -3.1953125,
    -1.7998046875,
    -1.873046875,
    -2.15234375,
    -1.9716796875,
    -2.681640625,
    -0.414306640625,
    -1.427734375,
    -2.94140625,
    -3.0625,
    -2.65234375,
    -2.736328125,
    -2.876953125,
    -2.251953125,
    -2.220703125,
    -1.63671875,
    -2.109375,
    -1.2958984375,
    -1.099609375,
    -1.4638671875,
    -1.5673828125,
    -1.642578125,
    -0.11541748046875,
    -1.4521484375,
    -2.451171875,
    -1.2548828125,
    -1.2080078125,
    -1.787109375,
    -1.5986328125,
    -1.3798828125,
    -1.2265625,
    -0.98291015625,
    -0.953125,
    -1.46484375,
    -1.3896484375,
    -1.361328125,
    -1.9814453125,
    -0.281494140625,
    -2.28515625,
    -2.0859375,
    -0.262451171875,
    -1.7900390625,
    -0.360595703125,
    -0.826171875,
    -0.192626953125,
    -0.953125,
    -2.16015625,
    -0.87939453125,
    -1.3896484375,
    -2.06640625,
    -1.4228515625,
    -1.4365234375,
    -1.20703125,
    -1.6630859375,
    -1.2734375,
    -1.14453125,
    -0.06719970703125,
    0.339599609375,
    -0.397705078125,
    1.033203125,
    1.0234375,
    0.0816650390625,
    -0.6083984375,
    -0.4443359375,
    0.031005859375,
    -0.81982421875,
    0.4296875,
    1.27734375,
    1.46484375,
    -0.52685546875,
    -0.2802734375,
    -0.90771484375,
    0.56982421875,
    0.69873046875,
    -0.66943359375,
    -0.04144287109375,
    -0.212158203125,
    0.5185546875,
    0.44775390625,
    -0.350830078125,
    -0.92431640625,
    1.7314453125,
    -0.2088623046875,
    -0.86279296875,
    -0.1978759765625,
    0.279052734375,
    -0.5732421875,
    -6.41796875
   ],
   "raw256_argmax": 1,
   "option_argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "p_full": [
    0.015146096237003803,
    0.9845128655433655,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0003410799545235932
   ],
   "p": [
    0.015151264145970345,
    0.9848487377166748
   ],
   "p_oracle": [
    0.015016984194517136,
    0.9849830269813538
   ],
   "abstain": 0.0003410799545235932,
   "abstain_oracle": 0.0003404226154088974,
   "abs_delta_abstain": 6.573391146957874e-07,
   "oracle_margin": 0.9699660539627075,
   "max_abs_delta_p": 0.00013428926467895508,
   "mean_abs_delta_p": 0.000134284608066082,
   "max_abs_delta_raw_logits": 0.006920337677001953,
   "mean_abs_delta_raw_logits": 0.001553524925839156,
   "logits_min": -7.42578125,
   "logits_max": 1.8935546875,
   "raw_logits_min": -7.42578125,
   "raw_logits_max": 1.8935546875,
   "raw_logits_absmax": 7.42578125,
   "oracle_logits_min": -7.428675651550293,
   "oracle_logits_max": 1.8974967002868652,
   "oracle_raw_logits_absmax": 7.428675651550293,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 0.4517637500539422,
   "seconds": 0.4517637500539422,
   "full_vocab_argmax_id": 1,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r17-slot",
   "request_id": "r17",
   "question_id": "slot",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 217,
   "nopts": 40,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -1.875,
    -1.814453125,
    -0.18115234375,
    -4.359375,
    -2.05078125,
    -2.86328125,
    -4.48046875,
    -2.654296875,
    -5.1796875,
    -2.970703125,
    -3.587890625,
    -3.615234375,
    -6.07421875,
    -5.21484375,
    -4.30859375,
    -3.525390625,
    -3.615234375,
    -5.84765625,
    -3.84765625,
    -4.78125,
    -5.0390625,
    -3.6875,
    -5.625,
    -5.59375,
    -4.16796875,
    -2.318359375,
    -3.29296875,
    5.30859375,
    -4.2578125,
    -6.58203125,
    -3.61328125,
    -7.24609375,
    -4.0703125,
    -5.74609375,
    -4.59375,
    -5.94140625,
    -7.25390625,
    -7.40234375,
    -7.13671875,
    -4.04296875,
    -6.56640625,
    -6.2265625,
    -6.12890625,
    -6.82421875,
    -4.92578125,
    -6.28125,
    -4.71484375,
    -4.58203125,
    -5.55078125,
    -6.27734375,
    -4.8125,
    -5.75,
    -5.12890625,
    -4.59375,
    -4.6640625,
    -4.6640625,
    -4.66796875,
    -4.7421875,
    -5.3515625,
    -6.609375,
    -8.0625,
    -7.53515625,
    -8.984375,
    -6.59765625,
    -4.6953125,
    -8.65625,
    -8.953125,
    -2.5703125,
    -9.9765625,
    -5.296875,
    -8.6640625,
    -5.8828125,
    -2.861328125,
    -7.203125,
    -3.712890625,
    -6.765625,
    -4.61328125,
    -7.7890625,
    -4.36328125,
    -6.01953125,
    -3.6171875,
    -4.91796875,
    -3.6640625,
    -4.6484375,
    -4.2109375,
    -5.453125,
    -3.3046875,
    -4.95703125,
    -3.69921875,
    -5.58203125,
    -4.96875,
    -3.873046875,
    -4.09375,
    -5.140625,
    -5.59375,
    -5.65234375,
    -5.98828125,
    -3.5390625,
    -3.4296875,
    -5.80859375,
    -6.9921875,
    -5.03125,
    -3.58203125,
    -5.74609375,
    -5.50390625,
    -7.265625,
    -4.4140625,
    -6.04296875,
    -4.4609375,
    -7.4765625,
    -5.28125,
    -5.73046875,
    -5.97265625,
    -6.51171875,
    -5.2109375,
    -2.8203125,
    -5.08203125,
    -6.96875,
    -5.5859375,
    -4.140625,
    -4.28125,
    -4.2421875,
    -3.12890625,
    -3.302734375,
    -2.708984375,
    -6.21484375,
    -7.1796875,
    -3.234375,
    -2.970703125,
    -6.05078125,
    -5.234375,
    -2.61328125,
    -6.46484375,
    -7.01171875,
    -5.453125,
    -5.671875,
    -3.857421875,
    -5.24609375,
    -7.5703125,
    -3.501953125,
    -3.123046875,
    -4.328125,
    -4.01953125,
    -2.63671875,
    -4.125,
    -4.73828125,
    -6.91796875,
    -6.53515625,
    -6.16015625,
    -5.09375,
    -4.51953125,
    -3.27734375,
    -1.2900390625,
    -2.685546875,
    -1.703125,
    -2.009765625,
    -1.9345703125,
    -1.234375,
    -3.111328125,
    0.33740234375,
    -1.306640625,
    -1.62109375,
    -1.0859375,
    -1.7958984375,
    -2.455078125,
    -1.759765625,
    -3.5234375,
    -1.7412109375,
    -5.19140625,
    -2.677734375,
    -3.50390625,
    -2.302734375,
    -1.9892578125,
    -3.646484375,
    -1.50390625,
    -3.34765625,
    -3.25,
    -3.390625,
    -3.314453125,
    -4.24609375,
    -4.1796875,
    -2.740234375,
    -2.517578125,
    -3.083984375,
    -2.88671875,
    -2.59765625,
    -2.966796875,
    -1.830078125,
    -3.287109375,
    -2.64453125,
    -2.48828125,
    -2.26953125,
    -3.486328125,
    -2.857421875,
    -1.818359375,
    -1.712890625,
    -1.8583984375,
    -2.205078125,
    -2.171875,
    -1.060546875,
    -1.15625,
    -3.193359375,
    -1.609375,
    -1.78515625,
    -2.83203125,
    -1.5693359375,
    -2.3359375,
    -3.056640625,
    0.1815185546875,
    -2.54296875,
    -1.0068359375,
    -1.263671875,
    -1.0810546875,
    -2.001953125,
    -4.06640625,
    -2.01171875,
    -1.3134765625,
    -2.392578125,
    -1.2353515625,
    -2.029296875,
    -1.7548828125,
    -2.248046875,
    -2.21875,
    -0.97607421875,
    0.471435546875,
    -0.5966796875,
    -0.070556640625,
    0.35888671875,
    0.8896484375,
    -1.2548828125,
    0.2548828125,
    0.259765625,
    0.043853759765625,
    -0.70703125,
    0.14501953125,
    1.3681640625,
    1.3408203125,
    -0.2469482421875,
    0.404052734375,
    -1.0048828125,
    0.288818359375,
    0.6142578125,
    -0.180419921875,
    -0.0248565673828125,
    0.2001953125,
    -0.01141357421875,
    0.4150390625,
    -0.019775390625,
    -1.2705078125,
    2.15625,
    -0.81494140625,
    0.6298828125,
    -0.472900390625,
    0.110107421875,
    0.171630859375,
    -4.46875
   ],
   "raw256_argmax": 27,
   "option_argmax": 27,
   "oracle_argmax": 27,
   "argmax_agrees": true,
   "p_full": [
    0.0011131871724501252,
    0.001178716542199254,
    0.0055146608501672745,
    0.00010648314491845667,
    0.0009428635239601135,
    0.0004376196302473545,
    9.497253631707281e-05,
    0.0005331369466148317,
    4.905954119749367e-05,
    0.0003953884297516197,
    0.00022070111299399287,
    0.00021507313067559153,
    2.107249565597158e-05,
    4.745690603158437e-05,
    0.00011171593359904364,
    0.00023412455630023032,
    0.00021507313067559153,
    2.6101735784322955e-05,
    0.00017267445218749344,
    7.148156146286055e-05,
    5.602978490060195e-05,
    0.00020088013843633235,
    3.221222505089827e-05,
    3.317736991448328e-05,
    0.00012758822413161397,
    0.0007322629680857062,
    0.000291611795546487,
    0.9859339594841003,
    0.00011720586917363107,
    1.3042797945672646e-05,
    0.00021547019423451275,
    6.9649740908062086e-06,
    0.00013991899322718382,
    2.8730206395266578e-05,
    8.533384243492037e-05,
    2.3889426302048378e-05,
    6.913765446370235e-06,
    6.009155640640529e-06,
    7.72313796915114e-06,
    0.00014358048792928457,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    9.602982754586264e-05
   ],
   "p": [
    0.0011132939253002405,
    0.001178829581476748,
    0.005515189841389656,
    0.00010649336036294699,
    0.000942953978665173,
    0.0004376615979708731,
    9.498164581600577e-05,
    0.0005331880529411137,
    4.906424510409124e-05,
    0.0003954263520427048,
    0.00022072228603065014,
    0.00021509376529138535,
    2.107451655319892e-05,
    4.746145714307204e-05,
    0.00011172665108460933,
    0.0002341470099054277,
    0.00021509376529138535,
    2.6104238713742234e-05,
    0.00017269101226702332,
    7.148841541493312e-05,
    5.603515819530003e-05,
    0.0002008994051720947,
    3.221531369490549e-05,
    3.3180553145939484e-05,
    0.00012760046229232103,
    0.0007323332247324288,
    0.0002916397643275559,
    0.9860285520553589,
    0.00011721711052814499,
    1.3044048500887584e-05,
    0.00021549085795413703,
    6.965642114664661e-06,
    0.00013993241009302437,
    2.873296216421295e-05,
    8.534202788723633e-05,
    2.3891718228696845e-05,
    6.914428468007827e-06,
    6.009731805534102e-06,
    7.723878297838382e-06,
    0.0001435942540410906
   ],
   "p_oracle": [
    0.0011019704397767782,
    0.0011741779744625092,
    0.005572291556745768,
    0.0001043995944201015,
    0.0009337730007246137,
    0.00043312847265042365,
    9.309892629971728e-05,
    0.0005304862861521542,
    4.843987335334532e-05,
    0.00039639012538827956,
    0.00021776325593236834,
    0.0002123292360920459,
    2.0675977793871425e-05,
    4.703338709077798e-05,
    0.00011001290840795264,
    0.0002325861423742026,
    0.0002133888629032299,
    2.58428535744315e-05,
    0.00017119513358920813,
    7.047851977404207e-05,
    5.554829476750456e-05,
    0.0001989987795241177,
    3.199079947080463e-05,
    3.3029155019903556e-05,
    0.00012720674567390233,
    0.0007189783500507474,
    0.0002877834776882082,
    0.9860422611236572,
    0.00011699549941113219,
    1.294876892643515e-05,
    0.0002151035296265036,
    6.854175808257423e-06,
    0.00014207363710738719,
    2.838669388438575e-05,
    8.500603871652856e-05,
    2.3704154955339618e-05,
    6.847472377558006e-06,
    5.990691533952486e-06,
    7.608487067045644e-06,
    0.00014321567141450942
   ],
   "abstain": 9.602982754586264e-05,
   "abstain_oracle": 9.498046711087227e-05,
   "abs_delta_abstain": 1.0493604349903762e-06,
   "oracle_margin": 0.9804699420928955,
   "max_abs_delta_p": 5.7101715356111526e-05,
   "mean_abs_delta_p": 3.695954990234895e-06,
   "max_abs_delta_raw_logits": 0.023356914520263672,
   "mean_abs_delta_raw_logits": 0.00389069234370254,
   "logits_min": -9.9765625,
   "logits_max": 5.30859375,
   "raw_logits_min": -9.9765625,
   "raw_logits_max": 5.30859375,
   "raw_logits_absmax": 9.9765625,
   "oracle_logits_min": -9.988116264343262,
   "oracle_logits_max": 5.315891265869141,
   "oracle_raw_logits_absmax": 9.988116264343262,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 1.3909195410087705,
   "seconds": 1.3909195410087705,
   "full_vocab_argmax_id": 27,
   "full_vocab_argmax_is_label": true
  },
  {
   "id": "r18-slot",
   "request_id": "r18",
   "question_id": "slot",
   "type": "choice",
   "qtype_index": 0,
   "kind": "slot",
   "tokens": 1449,
   "nopts": 255,
   "temperature": 1.058533787727356,
   "raw_logits": [
    -5.13671875,
    -5.14453125,
    -7.3203125,
    -6.67578125,
    -7.55859375,
    -8.15625,
    -7.15625,
    -8.46875,
    -9.484375,
    -6.359375,
    -9.1953125,
    -9.2578125,
    -7.421875,
    -9.5,
    -9.140625,
    -8.140625,
    -10.2421875,
    -9.453125,
    -7.890625,
    -10.2578125,
    -9.3515625,
    -7.90234375,
    -10.3359375,
    -10.1796875,
    -11.265625,
    -12.0703125,
    -10.484375,
    -11.15625,
    -10.1640625,
    -10.4140625,
    -9.5703125,
    -13.34375,
    -11.4765625,
    -10.3984375,
    -9.8203125,
    -11.671875,
    -11.40625,
    -9.2578125,
    -9.6875,
    -7.7421875,
    -11.140625,
    -11.078125,
    -12.4609375,
    -10.28125,
    -10.3984375,
    -12.0234375,
    -10.0390625,
    -9.9375,
    -10.6484375,
    -10.6953125,
    -13.140625,
    -11.5234375,
    -12.484375,
    -11.125,
    -10.96875,
    -9.3359375,
    -9.453125,
    -8.2890625,
    -9.2734375,
    -7.10546875,
    -14.875,
    -13.765625,
    -15.0,
    -11.109375,
    -11.1484375,
    -18.265625,
    -16.890625,
    -9.046875,
    -17.28125,
    -11.0,
    -16.21875,
    -10.1328125,
    -7.30078125,
    -12.03125,
    -9.1640625,
    -15.0703125,
    -10.078125,
    -14.4296875,
    -8.453125,
    -12.0859375,
    -8.7421875,
    -10.8046875,
    -7.0234375,
    -12.6015625,
    -8.7109375,
    -10.0,
    -8.6171875,
    -9.9453125,
    -9.4375,
    -14.5546875,
    -11.7109375,
    -13.1484375,
    -10.1640625,
    -10.09375,
    -9.7109375,
    -11.546875,
    -9.75,
    -7.05859375,
    -11.921875,
    -9.90625,
    -16.09375,
    -8.671875,
    -8.8359375,
    -10.859375,
    -8.796875,
    -15.875,
    -11.0625,
    -12.3984375,
    -9.890625,
    -14.3515625,
    -8.0703125,
    -14.3515625,
    -14.4765625,
    -15.6484375,
    -10.3359375,
    -6.703125,
    -10.1953125,
    -15.296875,
    -12.9453125,
    -10.171875,
    -8.2421875,
    -7.0234375,
    -8.03125,
    -9.453125,
    -7.29296875,
    -14.96875,
    -15.4921875,
    -9.6484375,
    -8.9296875,
    -16.5,
    -12.1171875,
    -11.7265625,
    -16.265625,
    -16.046875,
    -8.21875,
    -15.328125,
    -10.6953125,
    -10.34375,
    -16.3125,
    -12.625,
    -9.828125,
    -11.6796875,
    -10.046875,
    -9.9765625,
    -9.171875,
    -10.453125,
    -16.265625,
    -14.453125,
    -11.3828125,
    -10.4375,
    -11.71875,
    -10.7109375,
    -10.8515625,
    -10.203125,
    -6.0078125,
    -5.8046875,
    -4.9375,
    -5.40625,
    -7.50390625,
    -9.0390625,
    -6.68359375,
    -6.46484375,
    -6.3125,
    -6.328125,
    -9.5546875,
    -3.634765625,
    -7.07421875,
    -7.25,
    -10.8984375,
    -8.2265625,
    -7.57421875,
    -7.55859375,
    -4.484375,
    -9.2890625,
    -7.44140625,
    -7.515625,
    -11.0,
    -9.734375,
    -11.375,
    -10.1484375,
    -10.9921875,
    -12.890625,
    -10.1328125,
    -10.8046875,
    -11.4921875,
    -9.828125,
    -9.96875,
    -10.1640625,
    -11.078125,
    -11.6953125,
    -11.3984375,
    -9.890625,
    -10.734375,
    -9.5,
    -11.375,
    -10.4453125,
    -11.4140625,
    -9.375,
    -9.8359375,
    -10.203125,
    -9.6484375,
    -10.5703125,
    -8.3984375,
    -11.1328125,
    -9.765625,
    -8.7890625,
    -10.2421875,
    -8.859375,
    -0.71728515625,
    -10.7734375,
    -8.90625,
    -9.8359375,
    -9.4140625,
    -9.140625,
    -9.375,
    -9.125,
    -10.3203125,
    -8.5625,
    -9.046875,
    -5.93359375,
    -7.08203125,
    -0.01508331298828125,
    -1.7861328125,
    -1.0888671875,
    1.55078125,
    -0.8447265625,
    0.137939453125,
    -0.4443359375,
    -0.75390625,
    0.271484375,
    0.87890625,
    1.505859375,
    -2.0078125,
    -0.11822509765625,
    1.19921875,
    0.75390625,
    0.355712890625,
    -0.58544921875,
    0.86767578125,
    0.39208984375,
    -0.014862060546875,
    -1.2626953125,
    -1.3330078125,
    1.1806640625,
    -0.269775390625,
    1.7109375,
    -1.1708984375,
    0.142822265625,
    -1.3623046875,
    -0.2188720703125,
    -1.4833984375,
    1.1357421875,
    -0.2069091796875,
    1.8173828125,
    -1.251953125,
    -4.75
   ],
   "raw256_argmax": 253,
   "option_argmax": 253,
   "oracle_argmax": 253,
   "argmax_agrees": true,
   "p_full": [
    0.00015092024113982916,
    0.00014981046842876822,
    1.9180704839527607e-05,
    3.526162981870584e-05,
    1.5314491975004785e-05,
    8.707563210919034e-06,
    2.2396283384296112e-05,
    6.481633135990705e-06,
    2.483101525285747e-06,
    4.754632755066268e-05,
    3.2628022381686606e-06,
    3.0757305466977414e-06,
    1.742590029607527e-05,
    2.4467178718623472e-06,
    3.4357999538769946e-06,
    8.837047971610446e-06,
    1.2136111990912468e-06,
    2.557500465627527e-06,
    1.1191206795047037e-05,
    1.1958286449953448e-06,
    2.815040488712839e-06,
    1.1068001185776666e-05,
    1.1107490536232945e-06,
    1.2874251069661113e-06,
    4.615134230334661e-07,
    2.1579332099008752e-07,
    9.654174846218666e-07,
    5.117513524055539e-07,
    1.306569743064756e-06,
    1.0317224905520561e-06,
    2.2894762423675274e-06,
    6.479969982819966e-08,
    3.781299824368034e-07,
    1.047064642989426e-06,
    1.8078667380905244e-06,
    3.144187985526514e-07,
    4.0410000679003133e-07,
    3.0757305466977414e-06,
    2.049540853477083e-06,
    1.287590384890791e-05,
    5.193613219489635e-07,
    5.509498350875219e-07,
    1.492012131620868e-07,
    1.1696422461682232e-06,
    1.047064642989426e-06,
    2.255640225712341e-07,
    1.4703390434078756e-06,
    1.618403643988131e-06,
    8.268062856586766e-07,
    7.909916917014925e-07,
    7.850745475934673e-08,
    3.6175060813548043e-07,
    1.459339955545147e-07,
    5.270844667393249e-07,
    6.109227115302929e-07,
    2.856901573977666e-06,
    2.557500465627527e-06,
    7.680800081288908e-06,
    3.0306632652354892e-06,
    2.3496881112805568e-05,
    1.5252201279736255e-08,
    4.3499689184045565e-08,
    1.3553381528197406e-08,
    5.349224352357851e-07,
    5.155422968528001e-07,
    6.197682367314883e-10,
    2.2717665526528208e-09,
    3.7539764434768585e-06,
    1.5707188882885248e-09,
    5.931507871537178e-07,
    4.28568114330119e-09,
    1.3457173508868436e-06,
    1.9537888874765486e-05,
    2.2390536003058514e-07,
    3.3605622320465045e-06,
    1.2682356498316949e-08,
    1.4170689155434957e-06,
    2.322925674036469e-08,
    6.578017746505793e-06,
    2.1263139160510036e-07,
    5.0060912144545e-06,
    7.133417625482252e-07,
    2.539019988034852e-05,
    1.3064021686659544e-07,
    5.1560841711761896e-06,
    1.5256131291607744e-06,
    5.633569799101679e-06,
    1.60650301950227e-06,
    2.595531668703188e-06,
    2.0641936160359364e-08,
    3.0302743425636436e-07,
    7.793016720825108e-08,
    1.306569743064756e-06,
    1.3963051515020197e-06,
    2.004659791055019e-06,
    3.5382893770474766e-07,
    1.9320314095239155e-06,
    2.4560773454140872e-05,
    2.482782974766451e-07,
    1.6668943771946942e-06,
    4.822860333320023e-09,
    5.349910679797176e-06,
    4.581789198709885e-06,
    6.774239409423899e-07,
    4.754027486342238e-06,
    5.929979618457537e-09,
    5.591426202045113e-07,
    1.582759097118469e-07,
    1.6916817457968136e-06,
    2.5008541015836272e-08,
    9.443977432965767e-06,
    2.5008541015836272e-08,
    2.2223041185043257e-08,
    7.345264840807886e-09,
    1.1107490536232945e-06,
    3.4362405131105334e-05,
    1.2685611636698013e-06,
    1.0238720982158611e-08,
    9.44155473803221e-08,
    1.29696218209574e-06,
    8.028570846363436e-06,
    2.539019988034852e-05,
    9.79899414232932e-06,
    2.557500465627527e-06,
    1.968262040463742e-05,
    1.3959469136182179e-08,
    8.51360582032612e-09,
    2.1265866507746978e-06,
    4.1934463297366165e-06,
    3.285707483868805e-09,
    2.064458470840691e-07,
    2.985873379657278e-07,
    4.100039419085988e-09,
    5.041230100033545e-09,
    8.208317922253627e-06,
    9.940872125469014e-09,
    7.909916917014925e-07,
    1.102581222767185e-06,
    3.922438818193541e-09,
    1.2777944391473284e-07,
    1.7945729950952227e-06,
    3.1210677775561635e-07,
    1.4595271977668745e-06,
    1.559769088999019e-06,
    3.3358510336256586e-06,
    9.943432814907283e-07,
    4.100039419085988e-09,
    2.2720579195834034e-08,
    4.1314714849249867e-07,
    1.009129618978477e-06,
    3.007992006587301e-07,
    7.794016596562869e-07,
    6.824421916462597e-07,
    1.259233044947905e-06,
    6.627593393204734e-05,
    8.029597665881738e-05,
    0.00018217256001662463,
    0.00011699424794642255,
    1.6126483387779444e-05,
    3.7817851534782676e-06,
    3.500233651720919e-05,
    4.303733294364065e-05,
    4.969913061358966e-05,
    4.8970912757795304e-05,
    2.3235218122863444e-06,
    0.0006236888002604246,
    2.4200895495596342e-05,
    2.049803879344836e-05,
    6.52880999041372e-07,
    8.147959306370467e-06,
    1.5090095075720455e-05,
    1.5314491975004785e-05,
    0.0002795056498143822,
    2.9862562769267242e-06,
    1.7107326129917055e-05,
    1.594893001310993e-05,
    5.931507871537178e-07,
    1.9607614376582205e-06,
    4.16207626585674e-07,
    1.3259990510050557e-06,
    5.975446697448206e-07,
    9.942157674913688e-08,
    1.3457173508868436e-06,
    7.133417625482252e-07,
    3.725893975570216e-07,
    1.7945729950952227e-06,
    1.5713235370640177e-06,
    1.306569743064756e-06,
    5.509498350875219e-07,
    3.075336110214266e-07,
    4.0709346649236977e-07,
    1.6916817457968136e-06,
    7.623341957696539e-07,
    2.4467178718623472e-06,
    4.16207626585674e-07,
    1.0017091653935495e-06,
    4.011285170690826e-07,
    2.753396529442398e-06,
    1.7813769090935239e-06,
    1.259233044947905e-06,
    2.1265866507746978e-06,
    8.901369596969744e-07,
    6.926792593731079e-06,
    5.232086550677195e-07,
    1.9037220226891804e-06,
    4.789244485436939e-06,
    1.2136111990912468e-06,
    4.4814528337155934e-06,
    0.00981634110212326,
    7.347150017267268e-07,
    4.2873307393165305e-06,
    1.7813769090935239e-06,
    2.6536413315625396e-06,
    3.4357999538769946e-06,
    2.753396529442398e-06,
    3.4868917282437906e-06,
    1.1272662732153549e-06,
    5.932268322794698e-06,
    3.7539764434768585e-06,
    7.108962745405734e-05,
    2.4022940124268644e-05,
    0.019056735560297966,
    0.0035762148909270763,
    0.006910305004566908,
    0.08365469425916672,
    0.008702881634235382,
    0.022020673379302025,
    0.012703838758170605,
    0.009482542052865028,
    0.024981660768389702,
    0.04434389993548393,
    0.08017883449792862,
    0.002900501014664769,
    0.017287475988268852,
    0.060013849288225174,
    0.03940478339791298,
    0.027050700038671494,
    0.011118320748209953,
    0.04387592524290085,
    0.02799646370112896,
    0.019060717895627022,
    0.0058637987822294235,
    0.005486954003572464,
    0.05897105112671852,
    0.014981442131102085,
    0.0973193347454071,
    0.006395014468580484,
    0.02212248556315899,
    0.005337176378816366,
    0.015719478949904442,
    0.004760244395583868,
    0.05652080848813057,
    0.015898140147328377,
    0.10761464387178421,
    0.005923611111938953,
    0.00021747546270489693
   ],
   "p": [
    0.00015095308481249958,
    0.00014984306471887976,
    1.9184879420208745e-05,
    3.526930231601e-05,
    1.531782436359208e-05,
    8.709457688382827e-06,
    2.240115645690821e-05,
    6.483043762273155e-06,
    2.4836417651386e-06,
    4.755667396239005e-05,
    3.2635123261570698e-06,
    3.0763999347982462e-06,
    1.7429692888981663e-05,
    2.4472503810102353e-06,
    3.436547558521852e-06,
    8.838970643409993e-06,
    1.2138752936152741e-06,
    2.558057076385012e-06,
    1.1193642421858385e-05,
    1.1960888741668896e-06,
    2.815653033394483e-06,
    1.107040952774696e-05,
    1.1109907518402906e-06,
    1.2877052313342574e-06,
    4.616138369328837e-07,
    2.1584027365406655e-07,
    9.656275778979762e-07,
    5.118627086631022e-07,
    1.3068540738458978e-06,
    1.0319470220565563e-06,
    2.2899744180904236e-06,
    6.481379699607714e-08,
    3.782122632856044e-07,
    1.04729247141222e-06,
    1.8082600945490412e-06,
    3.144872096072504e-07,
    4.04187943559009e-07,
    3.0763999347982462e-06,
    2.0499867332546273e-06,
    1.2878706002084073e-05,
    5.194743266656587e-07,
    5.510697178578994e-07,
    1.4923368496511102e-07,
    1.169896790997882e-06,
    1.04729247141222e-06,
    2.256131068634204e-07,
    1.4706589581692242e-06,
    1.6187558458113926e-06,
    8.269861950793711e-07,
    7.911638135738031e-07,
    7.85245362067144e-08,
    3.6182933627060265e-07,
    1.4596575681480317e-07,
    5.27199176758586e-07,
    6.110556682870083e-07,
    2.8575232136063278e-06,
    2.558057076385012e-06,
    7.682471732550766e-06,
    3.0313226488942746e-06,
    2.3501994292018935e-05,
    1.5255519514312255e-08,
    4.350915361328589e-08,
    1.355633116872923e-08,
    5.35038850557612e-07,
    5.156545057616313e-07,
    6.19903073317829e-10,
    2.272260823943384e-09,
    3.754793397092726e-06,
    1.5710607259578069e-09,
    5.932798785579507e-07,
    4.286613730641875e-09,
    1.3460102081808145e-06,
    1.9542139853001572e-05,
    2.2395408905140357e-07,
    3.36129346578673e-06,
    1.2685116068666957e-08,
    1.4173772342473967e-06,
    2.3234312251929623e-08,
    6.579449291166384e-06,
    2.1267766214805306e-07,
    5.007180334359873e-06,
    7.134970019251341e-07,
    2.5395724151167087e-05,
    1.3066863857602584e-07,
    5.1572060328908265e-06,
    1.5259450947269215e-06,
    5.6347957979596686e-06,
    1.606852606528264e-06,
    2.596096464912989e-06,
    2.0646428566806208e-08,
    3.030933726222429e-07,
    7.794712786335367e-08,
    1.3068540738458978e-06,
    1.3966090364192496e-06,
    2.0050961211381946e-06,
    3.539059321155946e-07,
    1.93245182344981e-06,
    2.456611764500849e-05,
    2.483323271462723e-07,
    1.667257151893864e-06,
    4.823909716122898e-09,
    5.351074833015446e-06,
    4.582786004903028e-06,
    6.77571335927496e-07,
    4.755062036565505e-06,
    5.931270141701361e-09,
    5.592642651208735e-07,
    1.5831035682367656e-07,
    1.6920498637773562e-06,
    2.5013983773192194e-08,
    9.446032890991773e-06,
    2.5013983773192194e-08,
    2.2227876428360105e-08,
    7.346863117874136e-09,
    1.1109907518402906e-06,
    3.436988117755391e-05,
    1.2688371953117894e-06,
    1.0240948533635219e-08,
    9.443608917081292e-08,
    1.297244352826965e-06,
    8.030317985685542e-06,
    2.5395724151167087e-05,
    9.801125997910276e-06,
    2.558057076385012e-06,
    1.968690412468277e-05,
    1.3962506706377553e-08,
    8.515458560509614e-09,
    2.1270493562042248e-06,
    4.194359007669846e-06,
    3.2864224674966636e-09,
    2.0649076759582385e-07,
    2.986523099934857e-07,
    4.100931594308577e-09,
    5.042327000381874e-09,
    8.21010416984791e-06,
    9.943035728099403e-09,
    7.911638135738031e-07,
    1.1028211019947776e-06,
    3.923292357654873e-09,
    1.278072545574105e-07,
    1.7949635093827965e-06,
    3.12174705641155e-07,
    1.4598448387914686e-06,
    1.5601085578964557e-06,
    3.336577037771349e-06,
    9.945596275429125e-07,
    4.100931594308577e-09,
    2.2725522796918085e-08,
    4.1323704635942704e-07,
    1.0093492619489552e-06,
    3.0086465585554834e-07,
    7.795712804181676e-07,
    6.825906666563242e-07,
    1.259507030226814e-06,
    6.629035488003865e-05,
    8.031344623304904e-05,
    0.0001822121994337067,
    0.00011701970652211457,
    1.6129992218338884e-05,
    3.7826080188096967e-06,
    3.500995444483124e-05,
    4.30466971010901e-05,
    4.970994632458314e-05,
    4.8981568397721276e-05,
    2.32402749134053e-06,
    0.0006238245405256748,
    2.4206161469919607e-05,
    2.0502498955465853e-05,
    6.530230507451051e-07,
    8.149731911544222e-06,
    1.5093378351593856e-05,
    1.531782436359208e-05,
    0.00027956647682003677,
    2.986906110891141e-06,
    1.7111049601226114e-05,
    1.5952400644891895e-05,
    5.932798785579507e-07,
    1.9611882180470275e-06,
    4.162982065736287e-07,
    1.3262875881991931e-06,
    5.976746706437552e-07,
    9.944321277544077e-08,
    1.3460102081808145e-06,
    7.134970019251341e-07,
    3.7267048469402653e-07,
    1.7949635093827965e-06,
    1.5716655070718843e-06,
    1.3068540738458978e-06,
    5.510697178578994e-07,
    3.076005441471352e-07,
    4.0718205696066434e-07,
    1.6920498637773562e-06,
    7.625000648658897e-07,
    2.4472503810102353e-06,
    4.162982065736287e-07,
    1.0019271030614618e-06,
    4.0121580013874336e-07,
    2.753995659077191e-06,
    1.7817645812101546e-06,
    1.259507030226814e-06,
    2.1270493562042248e-06,
    8.90330682068452e-07,
    6.928300081199268e-06,
    5.233225124356977e-07,
    1.904136297525838e-06,
    4.7902867663651705e-06,
    1.2138752936152741e-06,
    4.482427812035894e-06,
    0.009818477556109428,
    7.348749022639822e-07,
    4.2882638808805496e-06,
    1.7817645812101546e-06,
    2.6542188606981654e-06,
    3.436547558521852e-06,
    2.753995659077191e-06,
    3.4876504741987446e-06,
    1.1275116094111581e-06,
    5.933559350523865e-06,
    3.754793397092726e-06,
    7.110509613994509e-05,
    2.4028167899814434e-05,
    0.019060881808400154,
    0.0035769930109381676,
    0.006911808624863625,
    0.083672896027565,
    0.008704775013029575,
    0.022025465965270996,
    0.012706602923572063,
    0.009484605863690376,
    0.024987095966935158,
    0.04435354843735695,
    0.08019628375768661,
    0.002901132218539715,
    0.0172912385314703,
    0.06002691015601158,
    0.03941335901618004,
    0.027056585997343063,
    0.011120740324258804,
    0.043885473161935806,
    0.028002556413412094,
    0.01906486600637436,
    0.005865074694156647,
    0.005488147959113121,
    0.05898388475179672,
    0.01498470176011324,
    0.09734050929546356,
    0.00639640586450696,
    0.022127298638224602,
    0.005338337738066912,
    0.01572289876639843,
    0.0047612800262868404,
    0.05653310939669609,
    0.0159015990793705,
    0.10763806104660034,
    0.005924900062382221
   ],
   "p_oracle": [
    0.0001469101116526872,
    0.0001461399078834802,
    1.8727199858403765e-05,
    3.445339825702831e-05,
    1.491959483246319e-05,
    8.525082193955313e-06,
    2.203234726039227e-05,
    6.327398750727298e-06,
    2.4345908968825825e-06,
    4.6558663598261774e-05,
    3.151977125526173e-06,
    2.988307187479222e-06,
    1.7149484847323038e-05,
    2.3426230200129794e-06,
    3.3076789804908913e-06,
    8.558708032069262e-06,
    1.175259626506886e-06,
    2.548649490563548e-06,
    1.0875735824811272e-05,
    1.174447334051365e-06,
    2.7018331820727326e-06,
    1.073516978067346e-05,
    1.0747947953859693e-06,
    1.2419112636052887e-06,
    4.439075667050929e-07,
    2.042345670361101e-07,
    9.404049592376396e-07,
    4.902366299575078e-07,
    1.2479507631724118e-06,
    9.923292054736521e-07,
    2.2280037228483707e-06,
    5.986074569364064e-08,
    3.658147988971905e-07,
    1.0361229669797467e-06,
    1.7353581824863795e-06,
    3.016077982920251e-07,
    3.907217376308836e-07,
    2.925652097474085e-06,
    1.966859144886257e-06,
    1.2392310054565314e-05,
    5.118952799421095e-07,
    5.359339070309943e-07,
    1.4427850203446724e-07,
    1.123934225688572e-06,
    1.0238458116873517e-06,
    2.1720221354826208e-07,
    1.4275401554186828e-06,
    1.5700845779065276e-06,
    8.077286679508688e-07,
    7.595568831675337e-07,
    7.496300469256312e-08,
    3.4219951317027153e-07,
    1.4025690120433865e-07,
    5.061549472884508e-07,
    5.872788619853964e-07,
    2.820014060489484e-06,
    2.5274971449107397e-06,
    7.469083357136697e-06,
    2.887191612899187e-06,
    2.2984444512985647e-05,
    1.4499901723752373e-08,
    4.224759564408487e-08,
    1.3127833931037003e-08,
    5.267225446914381e-07,
    5.032041485719674e-07,
    5.929003399351984e-10,
    2.150327471639457e-09,
    3.6878504943160806e-06,
    1.5137613385007853e-09,
    5.771068458670925e-07,
    4.05874622799729e-09,
    1.283262236029259e-06,
    1.8895223547588103e-05,
    2.1562215124504291e-07,
    3.2647783427819377e-06,
    1.216329259534632e-08,
    1.3845196917827707e-06,
    2.249413455501781e-08,
    6.356727226375369e-06,
    2.0436063152828865e-07,
    4.80034123029327e-06,
    6.884560548314766e-07,
    2.4657274479977787e-05,
    1.261227424720346e-07,
    5.058232545707142e-06,
    1.4855725112283835e-06,
    5.549032721319236e-06,
    1.5243693951561e-06,
    2.5501326490484644e-06,
    1.9503026749134733e-08,
    2.894741442105442e-07,
    7.430752901882443e-08,
    1.2425402928784024e-06,
    1.359587031402043e-06,
    1.943861207109876e-06,
    3.3811633670666197e-07,
    1.8900060467785806e-06,
    2.3787959435139783e-05,
    2.389311362094304e-07,
    1.6118586927404976e-06,
    4.5472798859691466e-09,
    5.195611720409943e-06,
    4.406738753459649e-06,
    6.553700018230302e-07,
    4.563604306895286e-06,
    5.59060486793328e-09,
    5.400765417107323e-07,
    1.514987673090218e-07,
    1.6380289480366628e-06,
    2.4197854386898143e-08,
    9.32275997911347e-06,
    2.393656828303392e-08,
    2.1264302318968475e-08,
    6.9980532479974045e-09,
    1.081376012734836e-06,
    3.306658618384972e-05,
    1.222553464685916e-06,
    9.822396229708374e-09,
    9.009193746578603e-08,
    1.2590995766004198e-06,
    7.929240382509306e-06,
    2.503086216165684e-05,
    9.561872502672486e-06,
    2.482203854015097e-06,
    1.9406297724344768e-05,
    1.3104779483796847e-08,
    8.068131940319745e-09,
    2.05597734748153e-06,
    4.0540408008382656e-06,
    3.169926543478141e-09,
    1.9746653379115742e-07,
    2.882631235934241e-07,
    3.9420209319018795e-09,
    4.785592810918615e-09,
    8.075558071141131e-06,
    9.4671372963262e-09,
    7.656602178940375e-07,
    1.0820309626069502e-06,
    3.729021536003074e-09,
    1.2206049859742052e-07,
    1.735182763695775e-06,
    3.0206345513761335e-07,
    1.4078510730541893e-06,
    1.526253186057147e-06,
    3.32214199261216e-06,
    9.722509730636375e-07,
    3.903593004395134e-09,
    2.2041644953674222e-08,
    4.0664875200491224e-07,
    9.705481716082431e-07,
    2.8843609811701754e-07,
    7.440521017088031e-07,
    6.660162625848898e-07,
    1.2110650686736335e-06,
    6.550806574523449e-05,
    7.979811925906688e-05,
    0.0001818382297642529,
    0.00011510254262248054,
    1.574801717652008e-05,
    3.7079730645928066e-06,
    3.386830576346256e-05,
    4.226343662594445e-05,
    4.857313251704909e-05,
    4.7070567234186456e-05,
    2.2569599877897417e-06,
    0.0006120369071140885,
    2.3661097657168284e-05,
    2.0005791157018393e-05,
    6.244418386813777e-07,
    7.96031963545829e-06,
    1.4662005014542956e-05,
    1.4956691302359104e-05,
    0.0002772198058664799,
    2.891156555051566e-06,
    1.6640678950352594e-05,
    1.5442494259332307e-05,
    5.737019250773301e-07,
    1.9168603557773167e-06,
    3.9845309629527037e-07,
    1.2928065871165018e-06,
    5.787465511275514e-07,
    9.430754488448656e-08,
    1.3109068959238357e-06,
    6.849967917332833e-07,
    3.59871819455293e-07,
    1.7630660522627295e-06,
    1.5343573522841325e-06,
    1.2713866226476966e-06,
    5.288820261739602e-07,
    2.9492520070562023e-07,
    3.9059952428033284e-07,
    1.6423742863480584e-06,
    7.348767780968046e-07,
    2.3556704036309384e-06,
    4.0035379811342864e-07,
    9.642487839300884e-07,
    3.915128559128789e-07,
    2.6290808818885125e-06,
    1.7516496200187248e-06,
    1.2177035841887118e-06,
    2.065924718408496e-06,
    8.670426154822053e-07,
    6.833991392340977e-06,
    4.97766279750067e-07,
    1.818960640775913e-06,
    4.63209062218084e-06,
    1.1824243983937777e-06,
    4.360892944532679e-06,
    0.009755837731063366,
    7.05798640865396e-07,
    4.1220273487851955e-06,
    1.7139896044682246e-06,
    2.5792653559619794e-06,
    3.3125340905826306e-06,
    2.6990644528268604e-06,
    3.3724088552844478e-06,
    1.0858207133424003e-06,
    5.779492767032934e-06,
    3.6232527236279566e-06,
    6.950746319489554e-05,
    2.359280188102275e-05,
    0.018962303176522255,
    0.0035667235497385263,
    0.006895596627146006,
    0.08383864909410477,
    0.008625222370028496,
    0.022189911454916,
    0.012730501592159271,
    0.009463458321988583,
    0.024959232658147812,
    0.0444105863571167,
    0.08021362125873566,
    0.0028592399321496487,
    0.017242614179849625,
    0.060589153319597244,
    0.039358872920274734,
    0.02689761482179165,
    0.011007904075086117,
    0.044373709708452225,
    0.02796688862144947,
    0.0190079715102911,
    0.005807948764413595,
    0.005553585942834616,
    0.05795270577073097,
    0.014917745254933834,
    0.0977199524641037,
    0.006410952657461166,
    0.022017937153577805,
    0.005312574561685324,
    0.015614868141710758,
    0.004747352562844753,
    0.057182542979717255,
    0.01578008383512497,
    0.107459157705307,
    0.005929228384047747
   ],
   "abstain": 0.00021747546270489693,
   "abstain_oracle": 0.00022339480347000062,
   "abs_delta_abstain": 5.919340765103698e-06,
   "oracle_margin": 0.009739205241203308,
   "max_abs_delta_p": 0.0010311789810657501,
   "mean_abs_delta_p": 2.033067847958558e-05,
   "max_abs_delta_raw_logits": 0.08431243896484375,
   "mean_abs_delta_raw_logits": 0.030978551774751395,
   "logits_min": -18.265625,
   "logits_max": 1.8173828125,
   "raw_logits_min": -18.265625,
   "raw_logits_max": 1.8173828125,
   "raw_logits_absmax": 18.265625,
   "oracle_logits_min": -18.312931060791016,
   "oracle_logits_max": 1.8154594898223877,
   "oracle_raw_logits_absmax": 18.312931060791016,
   "finite": true,
   "nonconstant": true,
   "wall_seconds_contended": 8.228386791073717,
   "seconds": 8.228386791073717,
   "full_vocab_argmax_id": 253,
   "full_vocab_argmax_is_label": true
  }
 ],
 "sessions": [
  {
   "schema": "coreai-slot-readout-session/1",
   "session": "fp16-00-kit-a1",
   "mode": "fp16",
   "indices": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    12,
    13,
    14,
    15,
    16,
    17,
    18,
    19,
    20,
    21,
    22,
    23,
    24,
    25,
    26,
    27,
    28,
    29
   ],
   "pid": 11899,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_fp16.h16c.aimodelc",
   "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
   "started_at": "2026-09-22T19:26:31+00:00",
   "phase": "complete",
   "runtime_calls": 2382,
   "planned_runtime_calls": 2382,
   "result": "PASS",
   "load_wall_seconds_contended": 1.5443053330527619,
   "active_row": "r01-route",
   "active_step": 112,
   "reset_check": {
    "row": "r01-route",
    "identical": true,
    "max_abs_diff": 0.0,
    "first_raw_logits": [
     0.93798828125,
     -4.04296875,
     -1.77734375,
     -3.1484375,
     -3.025390625,
     -3.1171875,
     -3.091796875,
     -3.0546875,
     -4.25,
     -2.951171875,
     -3.03125,
     -2.72265625,
     -4.34375,
     -3.388671875,
     -3.658203125,
     -3.583984375,
     -3.96484375,
     -4.8984375,
     -3.572265625,
     -3.408203125,
     -3.673828125,
     -2.38671875,
     -5.4921875,
     -5.2109375,
     -3.927734375,
     -4.7734375,
     -3.419921875,
     -4.7265625,
     -4.140625,
     -5.67578125,
     -4.296875,
     -4.5234375,
     -5.80859375,
     -3.939453125,
     -3.271484375,
     -4.9296875,
     -5.73046875,
     -6.21484375,
     -4.6953125,
     -3.015625,
     -5.89453125,
     -4.79296875,
     -5.14453125,
     -5.94921875,
     -3.826171875,
     -4.9609375,
     -4.34375,
     -4.94921875,
     -5.390625,
     -4.69140625,
     -5.26953125,
     -4.62109375,
     -3.86328125,
     -5.46875,
     -3.400390625,
     -3.7734375,
     -3.9453125,
     -4.4375,
     -4.578125,
     -5.76953125,
     -7.4453125,
     -7.1953125,
     -8.125,
     -5.8125,
     -3.712890625,
     -7.5,
     -8.015625,
     -2.70703125,
     -7.85546875,
     -4.3515625,
     -7.98828125,
     -5.375,
     -2.759765625,
     -5.4140625,
     -3.76171875,
     -6.3671875,
     -4.109375,
     -7.03125,
     -4.02734375,
     -6.0546875,
     -4.65625,
     -4.25390625,
     -3.181640625,
     -4.75,
     -3.55859375,
     -4.8828125,
     -2.650390625,
     -4.10546875,
     -3.326171875,
     -4.47265625,
     -4.01171875,
     -4.15234375,
     -4.30859375,
     -3.6171875,
     -4.76953125,
     -4.1015625,
     -5.0859375,
     -3.162109375,
     -2.6640625,
     -4.7421875,
     -6.1640625,
     -4.94921875,
     -3.408203125,
     -4.33984375,
     -4.77734375,
     -5.9453125,
     -3.4765625,
     -5.23046875,
     -4.12890625,
     -6.015625,
     -4.171875,
     -4.95703125,
     -5.01953125,
     -5.78125,
     -4.6953125,
     -3.5,
     -4.77734375,
     -5.984375,
     -4.2578125,
     -3.68359375,
     -3.125,
     -4.28515625,
     -2.890625,
     -3.681640625,
     -2.197265625,
     -5.09765625,
     -5.828125,
     -2.65625,
     -2.80078125,
     -5.77734375,
     -5.67578125,
     -3.236328125,
     -5.8125,
     -6.62890625,
     -5.34765625,
     -4.94140625,
     -3.755859375,
     -4.33203125,
     -6.3515625,
     -2.697265625,
     -3.625,
     -4.32421875,
     -3.037109375,
     -2.794921875,
     -4.24609375,
     -4.91796875,
     -5.87890625,
     -5.78515625,
     -5.21484375,
     -4.51171875,
     -4.5546875,
     -2.853515625,
     -1.0400390625,
     -3.15234375,
     -1.9384765625,
     -1.4453125,
     -1.5673828125,
     -1.51953125,
     -2.634765625,
     -0.86865234375,
     -1.8623046875,
     -1.7509765625,
     -1.4150390625,
     -1.34765625,
     -2.693359375,
     -1.841796875,
     -2.9765625,
     -2.271484375,
     -3.876953125,
     -2.115234375,
     -2.2109375,
     -2.541015625,
     -1.865234375,
     -2.91796875,
     -0.94873046875,
     -1.951171875,
     -3.669921875,
     -3.4140625,
     -3.232421875,
     -2.56640625,
     -3.90625,
     -2.49609375,
     -3.078125,
     -2.763671875,
     -3.390625,
     -2.501953125,
     -2.0,
     -2.05078125,
     -2.541015625,
     -2.658203125,
     -1.2587890625,
     -2.01171875,
     -2.76953125,
     -1.8408203125,
     -2.048828125,
     -2.10546875,
     -2.076171875,
     -2.02734375,
     -2.1171875,
     -1.134765625,
     -1.3955078125,
     -2.607421875,
     -1.2646484375,
     -2.27734375,
     -2.55859375,
     -1.7783203125,
     -2.203125,
     -2.5,
     0.07110595703125,
     -2.3203125,
     -1.7548828125,
     -1.232421875,
     -0.10736083984375,
     -1.1083984375,
     -3.306640625,
     -1.12890625,
     -2.017578125,
     -2.140625,
     -1.548828125,
     -2.09765625,
     -1.2158203125,
     -1.3671875,
     -1.0185546875,
     -0.8388671875,
     -0.0960693359375,
     -0.44384765625,
     -0.76806640625,
     0.890625,
     0.74658203125,
     -0.3798828125,
     -0.0428466796875,
     -0.452392578125,
     0.1131591796875,
     -0.91845703125,
     0.08892822265625,
     1.0849609375,
     0.904296875,
     -0.081298828125,
     -0.1031494140625,
     -0.72021484375,
     0.312255859375,
     0.53759765625,
     -0.7060546875,
     0.048126220703125,
     -0.137939453125,
     0.408935546875,
     0.78076171875,
     -0.75927734375,
     -0.13330078125,
     2.06640625,
     -0.72412109375,
     -0.288818359375,
     -0.50244140625,
     -0.0106964111328125,
     -0.3037109375,
     -2.74609375
    ],
    "final_raw_logits": [
     0.93798828125,
     -4.04296875,
     -1.77734375,
     -3.1484375,
     -3.025390625,
     -3.1171875,
     -3.091796875,
     -3.0546875,
     -4.25,
     -2.951171875,
     -3.03125,
     -2.72265625,
     -4.34375,
     -3.388671875,
     -3.658203125,
     -3.583984375,
     -3.96484375,
     -4.8984375,
     -3.572265625,
     -3.408203125,
     -3.673828125,
     -2.38671875,
     -5.4921875,
     -5.2109375,
     -3.927734375,
     -4.7734375,
     -3.419921875,
     -4.7265625,
     -4.140625,
     -5.67578125,
     -4.296875,
     -4.5234375,
     -5.80859375,
     -3.939453125,
     -3.271484375,
     -4.9296875,
     -5.73046875,
     -6.21484375,
     -4.6953125,
     -3.015625,
     -5.89453125,
     -4.79296875,
     -5.14453125,
     -5.94921875,
     -3.826171875,
     -4.9609375,
     -4.34375,
     -4.94921875,
     -5.390625,
     -4.69140625,
     -5.26953125,
     -4.62109375,
     -3.86328125,
     -5.46875,
     -3.400390625,
     -3.7734375,
     -3.9453125,
     -4.4375,
     -4.578125,
     -5.76953125,
     -7.4453125,
     -7.1953125,
     -8.125,
     -5.8125,
     -3.712890625,
     -7.5,
     -8.015625,
     -2.70703125,
     -7.85546875,
     -4.3515625,
     -7.98828125,
     -5.375,
     -2.759765625,
     -5.4140625,
     -3.76171875,
     -6.3671875,
     -4.109375,
     -7.03125,
     -4.02734375,
     -6.0546875,
     -4.65625,
     -4.25390625,
     -3.181640625,
     -4.75,
     -3.55859375,
     -4.8828125,
     -2.650390625,
     -4.10546875,
     -3.326171875,
     -4.47265625,
     -4.01171875,
     -4.15234375,
     -4.30859375,
     -3.6171875,
     -4.76953125,
     -4.1015625,
     -5.0859375,
     -3.162109375,
     -2.6640625,
     -4.7421875,
     -6.1640625,
     -4.94921875,
     -3.408203125,
     -4.33984375,
     -4.77734375,
     -5.9453125,
     -3.4765625,
     -5.23046875,
     -4.12890625,
     -6.015625,
     -4.171875,
     -4.95703125,
     -5.01953125,
     -5.78125,
     -4.6953125,
     -3.5,
     -4.77734375,
     -5.984375,
     -4.2578125,
     -3.68359375,
     -3.125,
     -4.28515625,
     -2.890625,
     -3.681640625,
     -2.197265625,
     -5.09765625,
     -5.828125,
     -2.65625,
     -2.80078125,
     -5.77734375,
     -5.67578125,
     -3.236328125,
     -5.8125,
     -6.62890625,
     -5.34765625,
     -4.94140625,
     -3.755859375,
     -4.33203125,
     -6.3515625,
     -2.697265625,
     -3.625,
     -4.32421875,
     -3.037109375,
     -2.794921875,
     -4.24609375,
     -4.91796875,
     -5.87890625,
     -5.78515625,
     -5.21484375,
     -4.51171875,
     -4.5546875,
     -2.853515625,
     -1.0400390625,
     -3.15234375,
     -1.9384765625,
     -1.4453125,
     -1.5673828125,
     -1.51953125,
     -2.634765625,
     -0.86865234375,
     -1.8623046875,
     -1.7509765625,
     -1.4150390625,
     -1.34765625,
     -2.693359375,
     -1.841796875,
     -2.9765625,
     -2.271484375,
     -3.876953125,
     -2.115234375,
     -2.2109375,
     -2.541015625,
     -1.865234375,
     -2.91796875,
     -0.94873046875,
     -1.951171875,
     -3.669921875,
     -3.4140625,
     -3.232421875,
     -2.56640625,
     -3.90625,
     -2.49609375,
     -3.078125,
     -2.763671875,
     -3.390625,
     -2.501953125,
     -2.0,
     -2.05078125,
     -2.541015625,
     -2.658203125,
     -1.2587890625,
     -2.01171875,
     -2.76953125,
     -1.8408203125,
     -2.048828125,
     -2.10546875,
     -2.076171875,
     -2.02734375,
     -2.1171875,
     -1.134765625,
     -1.3955078125,
     -2.607421875,
     -1.2646484375,
     -2.27734375,
     -2.55859375,
     -1.7783203125,
     -2.203125,
     -2.5,
     0.07110595703125,
     -2.3203125,
     -1.7548828125,
     -1.232421875,
     -0.10736083984375,
     -1.1083984375,
     -3.306640625,
     -1.12890625,
     -2.017578125,
     -2.140625,
     -1.548828125,
     -2.09765625,
     -1.2158203125,
     -1.3671875,
     -1.0185546875,
     -0.8388671875,
     -0.0960693359375,
     -0.44384765625,
     -0.76806640625,
     0.890625,
     0.74658203125,
     -0.3798828125,
     -0.0428466796875,
     -0.452392578125,
     0.1131591796875,
     -0.91845703125,
     0.08892822265625,
     1.0849609375,
     0.904296875,
     -0.081298828125,
     -0.1031494140625,
     -0.72021484375,
     0.312255859375,
     0.53759765625,
     -0.7060546875,
     0.048126220703125,
     -0.137939453125,
     0.408935546875,
     0.78076171875,
     -0.75927734375,
     -0.13330078125,
     2.06640625,
     -0.72412109375,
     -0.288818359375,
     -0.50244140625,
     -0.0106964111328125,
     -0.3037109375,
     -2.74609375
    ],
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-22T19:26:47+00:00",
   "generated_at": "2026-09-22T19:26:47+00:00",
   "wall_seconds_contended": 16.22509829106275,
   "kind": "kit",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/fp16-00-kit-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/readout-fp16-00-kit-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/conversion/slot/readout_gate_slot.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/fp16-00-kit-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 20.358457583002746,
   "mpsgraph_scratch_before": {
    "at": "2026-09-22T19:26:27+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   },
   "mpsgraph_scratch_after": {
    "at": "2026-09-22T19:26:47+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   }
  },
  {
   "schema": "coreai-slot-readout-session/1",
   "session": "fp16-01-kit-a1",
   "mode": "fp16",
   "indices": [
    30,
    31,
    32,
    33,
    34,
    35,
    36,
    37,
    38,
    39,
    40,
    41,
    42,
    43,
    44,
    45,
    46,
    47
   ],
   "pid": 12773,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_fp16.h16c.aimodelc",
   "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
   "started_at": "2026-09-22T19:26:51+00:00",
   "phase": "complete",
   "runtime_calls": 1274,
   "planned_runtime_calls": 1274,
   "result": "PASS",
   "load_wall_seconds_contended": 0.2243432089453563,
   "active_row": "r11-status",
   "active_step": 36,
   "reset_check": {
    "row": "r11-status",
    "identical": true,
    "max_abs_diff": 0.0,
    "first_raw_logits": [
     -2.80859375,
     3.21484375,
     -2.43359375,
     -2.560546875,
     -2.181640625,
     -2.837890625,
     -2.73828125,
     -3.576171875,
     -3.716796875,
     -2.44921875,
     -2.994140625,
     -3.966796875,
     -5.328125,
     -3.14453125,
     -4.03125,
     -4.3515625,
     -3.69140625,
     -4.31640625,
     -2.91796875,
     -3.05859375,
     -3.412109375,
     -1.9912109375,
     -5.26953125,
     -6.859375,
     -3.91015625,
     -5.3671875,
     -2.41796875,
     -5.19140625,
     -4.4609375,
     -6.58203125,
     -4.45703125,
     -4.38671875,
     -6.1875,
     -4.89453125,
     -3.060546875,
     -5.35546875,
     -5.24609375,
     -5.87890625,
     -4.203125,
     -2.90625,
     -7.19921875,
     -4.29296875,
     -4.66015625,
     -6.0078125,
     -4.2578125,
     -5.046875,
     -4.5,
     -4.49609375,
     -5.36328125,
     -4.65625,
     -5.67578125,
     -4.67578125,
     -2.734375,
     -4.87109375,
     -2.66015625,
     -3.3359375,
     -3.52734375,
     -4.39453125,
     -4.70703125,
     -5.421875,
     -7.55078125,
     -6.3515625,
     -7.62890625,
     -6.51953125,
     -3.609375,
     -7.4375,
     -7.67578125,
     -2.3125,
     -7.74609375,
     -4.07421875,
     -7.7890625,
     -5.28125,
     -2.470703125,
     -6.16015625,
     -3.724609375,
     -6.17578125,
     -4.2109375,
     -6.68359375,
     -4.2421875,
     -5.65625,
     -4.23828125,
     -4.11328125,
     -2.673828125,
     -4.2265625,
     -3.513671875,
     -3.90234375,
     -2.58984375,
     -3.802734375,
     -2.998046875,
     -4.84375,
     -3.4375,
     -4.07421875,
     -4.203125,
     -4.20703125,
     -4.58203125,
     -4.6484375,
     -5.12890625,
     -3.751953125,
     -3.97265625,
     -4.13671875,
     -6.046875,
     -4.5,
     -3.73046875,
     -4.2109375,
     -4.47265625,
     -5.984375,
     -4.19921875,
     -5.59765625,
     -4.12890625,
     -5.5703125,
     -4.32421875,
     -4.58984375,
     -4.94140625,
     -6.453125,
     -4.74609375,
     -4.00390625,
     -4.5703125,
     -6.01953125,
     -4.4765625,
     -3.48828125,
     -3.373046875,
     -3.96875,
     -3.27734375,
     -3.78125,
     -2.365234375,
     -5.8203125,
     -5.7265625,
     -3.103515625,
     -3.095703125,
     -5.3046875,
     -5.55078125,
     -3.66796875,
     -5.62890625,
     -5.94921875,
     -4.91796875,
     -4.67578125,
     -3.921875,
     -4.94140625,
     -5.80078125,
     -2.88671875,
     -3.712890625,
     -4.13671875,
     -3.693359375,
     -3.330078125,
     -4.671875,
     -5.1875,
     -6.0859375,
     -5.1484375,
     -4.62109375,
     -4.37890625,
     -4.7421875,
     -3.1328125,
     -1.12890625,
     -3.41796875,
     -2.443359375,
     -1.58984375,
     -2.005859375,
     -1.9189453125,
     -3.41015625,
     -0.43701171875,
     -2.123046875,
     -1.9033203125,
     -1.66015625,
     -1.265625,
     -2.560546875,
     -2.30078125,
     -2.857421875,
     -1.7666015625,
     -3.42578125,
     -2.029296875,
     -2.298828125,
     -2.529296875,
     -2.4296875,
     -2.93359375,
     -0.84716796875,
     -1.9375,
     -3.193359375,
     -3.275390625,
     -3.12109375,
     -3.51171875,
     -3.388671875,
     -3.1640625,
     -2.6875,
     -2.189453125,
     -2.509765625,
     -1.6328125,
     -1.326171875,
     -1.755859375,
     -1.92578125,
     -2.087890625,
     -0.94677734375,
     -1.755859375,
     -2.861328125,
     -1.697265625,
     -1.3798828125,
     -2.15625,
     -2.04296875,
     -1.8681640625,
     -1.587890625,
     -1.4091796875,
     -1.3427734375,
     -1.943359375,
     -1.7138671875,
     -1.8994140625,
     -2.259765625,
     -0.97216796875,
     -2.62109375,
     -2.58984375,
     -0.38525390625,
     -2.181640625,
     -0.92431640625,
     -1.26953125,
     -0.771484375,
     -1.3740234375,
     -2.52734375,
     -1.2041015625,
     -1.7509765625,
     -2.328125,
     -1.8232421875,
     -1.912109375,
     -1.5244140625,
     -1.7099609375,
     -1.4599609375,
     -1.3232421875,
     -0.2054443359375,
     0.10260009765625,
     -0.4775390625,
     0.82421875,
     1.078125,
     -0.035430908203125,
     -0.8447265625,
     -0.478515625,
     0.02947998046875,
     -0.693359375,
     0.552734375,
     1.26953125,
     1.470703125,
     -0.71826171875,
     -0.0972900390625,
     -0.8984375,
     0.3876953125,
     0.8232421875,
     -0.83349609375,
     -0.177490234375,
     0.052093505859375,
     0.45751953125,
     0.316162109375,
     -0.45654296875,
     -0.95263671875,
     1.5244140625,
     -0.0187225341796875,
     -0.81201171875,
     -0.169677734375,
     0.419921875,
     -0.51611328125,
     -4.2109375
    ],
    "final_raw_logits": [
     -2.80859375,
     3.21484375,
     -2.43359375,
     -2.560546875,
     -2.181640625,
     -2.837890625,
     -2.73828125,
     -3.576171875,
     -3.716796875,
     -2.44921875,
     -2.994140625,
     -3.966796875,
     -5.328125,
     -3.14453125,
     -4.03125,
     -4.3515625,
     -3.69140625,
     -4.31640625,
     -2.91796875,
     -3.05859375,
     -3.412109375,
     -1.9912109375,
     -5.26953125,
     -6.859375,
     -3.91015625,
     -5.3671875,
     -2.41796875,
     -5.19140625,
     -4.4609375,
     -6.58203125,
     -4.45703125,
     -4.38671875,
     -6.1875,
     -4.89453125,
     -3.060546875,
     -5.35546875,
     -5.24609375,
     -5.87890625,
     -4.203125,
     -2.90625,
     -7.19921875,
     -4.29296875,
     -4.66015625,
     -6.0078125,
     -4.2578125,
     -5.046875,
     -4.5,
     -4.49609375,
     -5.36328125,
     -4.65625,
     -5.67578125,
     -4.67578125,
     -2.734375,
     -4.87109375,
     -2.66015625,
     -3.3359375,
     -3.52734375,
     -4.39453125,
     -4.70703125,
     -5.421875,
     -7.55078125,
     -6.3515625,
     -7.62890625,
     -6.51953125,
     -3.609375,
     -7.4375,
     -7.67578125,
     -2.3125,
     -7.74609375,
     -4.07421875,
     -7.7890625,
     -5.28125,
     -2.470703125,
     -6.16015625,
     -3.724609375,
     -6.17578125,
     -4.2109375,
     -6.68359375,
     -4.2421875,
     -5.65625,
     -4.23828125,
     -4.11328125,
     -2.673828125,
     -4.2265625,
     -3.513671875,
     -3.90234375,
     -2.58984375,
     -3.802734375,
     -2.998046875,
     -4.84375,
     -3.4375,
     -4.07421875,
     -4.203125,
     -4.20703125,
     -4.58203125,
     -4.6484375,
     -5.12890625,
     -3.751953125,
     -3.97265625,
     -4.13671875,
     -6.046875,
     -4.5,
     -3.73046875,
     -4.2109375,
     -4.47265625,
     -5.984375,
     -4.19921875,
     -5.59765625,
     -4.12890625,
     -5.5703125,
     -4.32421875,
     -4.58984375,
     -4.94140625,
     -6.453125,
     -4.74609375,
     -4.00390625,
     -4.5703125,
     -6.01953125,
     -4.4765625,
     -3.48828125,
     -3.373046875,
     -3.96875,
     -3.27734375,
     -3.78125,
     -2.365234375,
     -5.8203125,
     -5.7265625,
     -3.103515625,
     -3.095703125,
     -5.3046875,
     -5.55078125,
     -3.66796875,
     -5.62890625,
     -5.94921875,
     -4.91796875,
     -4.67578125,
     -3.921875,
     -4.94140625,
     -5.80078125,
     -2.88671875,
     -3.712890625,
     -4.13671875,
     -3.693359375,
     -3.330078125,
     -4.671875,
     -5.1875,
     -6.0859375,
     -5.1484375,
     -4.62109375,
     -4.37890625,
     -4.7421875,
     -3.1328125,
     -1.12890625,
     -3.41796875,
     -2.443359375,
     -1.58984375,
     -2.005859375,
     -1.9189453125,
     -3.41015625,
     -0.43701171875,
     -2.123046875,
     -1.9033203125,
     -1.66015625,
     -1.265625,
     -2.560546875,
     -2.30078125,
     -2.857421875,
     -1.7666015625,
     -3.42578125,
     -2.029296875,
     -2.298828125,
     -2.529296875,
     -2.4296875,
     -2.93359375,
     -0.84716796875,
     -1.9375,
     -3.193359375,
     -3.275390625,
     -3.12109375,
     -3.51171875,
     -3.388671875,
     -3.1640625,
     -2.6875,
     -2.189453125,
     -2.509765625,
     -1.6328125,
     -1.326171875,
     -1.755859375,
     -1.92578125,
     -2.087890625,
     -0.94677734375,
     -1.755859375,
     -2.861328125,
     -1.697265625,
     -1.3798828125,
     -2.15625,
     -2.04296875,
     -1.8681640625,
     -1.587890625,
     -1.4091796875,
     -1.3427734375,
     -1.943359375,
     -1.7138671875,
     -1.8994140625,
     -2.259765625,
     -0.97216796875,
     -2.62109375,
     -2.58984375,
     -0.38525390625,
     -2.181640625,
     -0.92431640625,
     -1.26953125,
     -0.771484375,
     -1.3740234375,
     -2.52734375,
     -1.2041015625,
     -1.7509765625,
     -2.328125,
     -1.8232421875,
     -1.912109375,
     -1.5244140625,
     -1.7099609375,
     -1.4599609375,
     -1.3232421875,
     -0.2054443359375,
     0.10260009765625,
     -0.4775390625,
     0.82421875,
     1.078125,
     -0.035430908203125,
     -0.8447265625,
     -0.478515625,
     0.02947998046875,
     -0.693359375,
     0.552734375,
     1.26953125,
     1.470703125,
     -0.71826171875,
     -0.0972900390625,
     -0.8984375,
     0.3876953125,
     0.8232421875,
     -0.83349609375,
     -0.177490234375,
     0.052093505859375,
     0.45751953125,
     0.316162109375,
     -0.45654296875,
     -0.95263671875,
     1.5244140625,
     -0.0187225341796875,
     -0.81201171875,
     -0.169677734375,
     0.419921875,
     -0.51611328125,
     -4.2109375
    ],
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-22T19:26:58+00:00",
   "generated_at": "2026-09-22T19:26:58+00:00",
   "wall_seconds_contended": 7.620699416962452,
   "kind": "kit",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/fp16-01-kit-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/readout-fp16-01-kit-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/conversion/slot/readout_gate_slot.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/fp16-01-kit-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 11.601954500074498,
   "mpsgraph_scratch_before": {
    "at": "2026-09-22T19:26:47+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   },
   "mpsgraph_scratch_after": {
    "at": "2026-09-22T19:26:59+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   }
  },
  {
   "schema": "coreai-slot-readout-session/1",
   "session": "fp16-02-wide-a1",
   "mode": "fp16",
   "indices": [
    48
   ],
   "pid": 13272,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_fp16.h16c.aimodelc",
   "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
   "started_at": "2026-09-22T19:27:02+00:00",
   "phase": "complete",
   "runtime_calls": 434,
   "planned_runtime_calls": 434,
   "result": "PASS",
   "load_wall_seconds_contended": 0.22450991696678102,
   "active_row": "r17-slot",
   "active_step": 216,
   "reset_check": {
    "row": "r17-slot",
    "identical": true,
    "max_abs_diff": 0.0,
    "first_raw_logits": [
     -1.875,
     -1.814453125,
     -0.18115234375,
     -4.359375,
     -2.05078125,
     -2.86328125,
     -4.48046875,
     -2.654296875,
     -5.1796875,
     -2.970703125,
     -3.587890625,
     -3.615234375,
     -6.07421875,
     -5.21484375,
     -4.30859375,
     -3.525390625,
     -3.615234375,
     -5.84765625,
     -3.84765625,
     -4.78125,
     -5.0390625,
     -3.6875,
     -5.625,
     -5.59375,
     -4.16796875,
     -2.318359375,
     -3.29296875,
     5.30859375,
     -4.2578125,
     -6.58203125,
     -3.61328125,
     -7.24609375,
     -4.0703125,
     -5.74609375,
     -4.59375,
     -5.94140625,
     -7.25390625,
     -7.40234375,
     -7.13671875,
     -4.04296875,
     -6.56640625,
     -6.2265625,
     -6.12890625,
     -6.82421875,
     -4.92578125,
     -6.28125,
     -4.71484375,
     -4.58203125,
     -5.55078125,
     -6.27734375,
     -4.8125,
     -5.75,
     -5.12890625,
     -4.59375,
     -4.6640625,
     -4.6640625,
     -4.66796875,
     -4.7421875,
     -5.3515625,
     -6.609375,
     -8.0625,
     -7.53515625,
     -8.984375,
     -6.59765625,
     -4.6953125,
     -8.65625,
     -8.953125,
     -2.5703125,
     -9.9765625,
     -5.296875,
     -8.6640625,
     -5.8828125,
     -2.861328125,
     -7.203125,
     -3.712890625,
     -6.765625,
     -4.61328125,
     -7.7890625,
     -4.36328125,
     -6.01953125,
     -3.6171875,
     -4.91796875,
     -3.6640625,
     -4.6484375,
     -4.2109375,
     -5.453125,
     -3.3046875,
     -4.95703125,
     -3.69921875,
     -5.58203125,
     -4.96875,
     -3.873046875,
     -4.09375,
     -5.140625,
     -5.59375,
     -5.65234375,
     -5.98828125,
     -3.5390625,
     -3.4296875,
     -5.80859375,
     -6.9921875,
     -5.03125,
     -3.58203125,
     -5.74609375,
     -5.50390625,
     -7.265625,
     -4.4140625,
     -6.04296875,
     -4.4609375,
     -7.4765625,
     -5.28125,
     -5.73046875,
     -5.97265625,
     -6.51171875,
     -5.2109375,
     -2.8203125,
     -5.08203125,
     -6.96875,
     -5.5859375,
     -4.140625,
     -4.28125,
     -4.2421875,
     -3.12890625,
     -3.302734375,
     -2.708984375,
     -6.21484375,
     -7.1796875,
     -3.234375,
     -2.970703125,
     -6.05078125,
     -5.234375,
     -2.61328125,
     -6.46484375,
     -7.01171875,
     -5.453125,
     -5.671875,
     -3.857421875,
     -5.24609375,
     -7.5703125,
     -3.501953125,
     -3.123046875,
     -4.328125,
     -4.01953125,
     -2.63671875,
     -4.125,
     -4.73828125,
     -6.91796875,
     -6.53515625,
     -6.16015625,
     -5.09375,
     -4.51953125,
     -3.27734375,
     -1.2900390625,
     -2.685546875,
     -1.703125,
     -2.009765625,
     -1.9345703125,
     -1.234375,
     -3.111328125,
     0.33740234375,
     -1.306640625,
     -1.62109375,
     -1.0859375,
     -1.7958984375,
     -2.455078125,
     -1.759765625,
     -3.5234375,
     -1.7412109375,
     -5.19140625,
     -2.677734375,
     -3.50390625,
     -2.302734375,
     -1.9892578125,
     -3.646484375,
     -1.50390625,
     -3.34765625,
     -3.25,
     -3.390625,
     -3.314453125,
     -4.24609375,
     -4.1796875,
     -2.740234375,
     -2.517578125,
     -3.083984375,
     -2.88671875,
     -2.59765625,
     -2.966796875,
     -1.830078125,
     -3.287109375,
     -2.64453125,
     -2.48828125,
     -2.26953125,
     -3.486328125,
     -2.857421875,
     -1.818359375,
     -1.712890625,
     -1.8583984375,
     -2.205078125,
     -2.171875,
     -1.060546875,
     -1.15625,
     -3.193359375,
     -1.609375,
     -1.78515625,
     -2.83203125,
     -1.5693359375,
     -2.3359375,
     -3.056640625,
     0.1815185546875,
     -2.54296875,
     -1.0068359375,
     -1.263671875,
     -1.0810546875,
     -2.001953125,
     -4.06640625,
     -2.01171875,
     -1.3134765625,
     -2.392578125,
     -1.2353515625,
     -2.029296875,
     -1.7548828125,
     -2.248046875,
     -2.21875,
     -0.97607421875,
     0.471435546875,
     -0.5966796875,
     -0.070556640625,
     0.35888671875,
     0.8896484375,
     -1.2548828125,
     0.2548828125,
     0.259765625,
     0.043853759765625,
     -0.70703125,
     0.14501953125,
     1.3681640625,
     1.3408203125,
     -0.2469482421875,
     0.404052734375,
     -1.0048828125,
     0.288818359375,
     0.6142578125,
     -0.180419921875,
     -0.0248565673828125,
     0.2001953125,
     -0.01141357421875,
     0.4150390625,
     -0.019775390625,
     -1.2705078125,
     2.15625,
     -0.81494140625,
     0.6298828125,
     -0.472900390625,
     0.110107421875,
     0.171630859375,
     -4.46875
    ],
    "final_raw_logits": [
     -1.875,
     -1.814453125,
     -0.18115234375,
     -4.359375,
     -2.05078125,
     -2.86328125,
     -4.48046875,
     -2.654296875,
     -5.1796875,
     -2.970703125,
     -3.587890625,
     -3.615234375,
     -6.07421875,
     -5.21484375,
     -4.30859375,
     -3.525390625,
     -3.615234375,
     -5.84765625,
     -3.84765625,
     -4.78125,
     -5.0390625,
     -3.6875,
     -5.625,
     -5.59375,
     -4.16796875,
     -2.318359375,
     -3.29296875,
     5.30859375,
     -4.2578125,
     -6.58203125,
     -3.61328125,
     -7.24609375,
     -4.0703125,
     -5.74609375,
     -4.59375,
     -5.94140625,
     -7.25390625,
     -7.40234375,
     -7.13671875,
     -4.04296875,
     -6.56640625,
     -6.2265625,
     -6.12890625,
     -6.82421875,
     -4.92578125,
     -6.28125,
     -4.71484375,
     -4.58203125,
     -5.55078125,
     -6.27734375,
     -4.8125,
     -5.75,
     -5.12890625,
     -4.59375,
     -4.6640625,
     -4.6640625,
     -4.66796875,
     -4.7421875,
     -5.3515625,
     -6.609375,
     -8.0625,
     -7.53515625,
     -8.984375,
     -6.59765625,
     -4.6953125,
     -8.65625,
     -8.953125,
     -2.5703125,
     -9.9765625,
     -5.296875,
     -8.6640625,
     -5.8828125,
     -2.861328125,
     -7.203125,
     -3.712890625,
     -6.765625,
     -4.61328125,
     -7.7890625,
     -4.36328125,
     -6.01953125,
     -3.6171875,
     -4.91796875,
     -3.6640625,
     -4.6484375,
     -4.2109375,
     -5.453125,
     -3.3046875,
     -4.95703125,
     -3.69921875,
     -5.58203125,
     -4.96875,
     -3.873046875,
     -4.09375,
     -5.140625,
     -5.59375,
     -5.65234375,
     -5.98828125,
     -3.5390625,
     -3.4296875,
     -5.80859375,
     -6.9921875,
     -5.03125,
     -3.58203125,
     -5.74609375,
     -5.50390625,
     -7.265625,
     -4.4140625,
     -6.04296875,
     -4.4609375,
     -7.4765625,
     -5.28125,
     -5.73046875,
     -5.97265625,
     -6.51171875,
     -5.2109375,
     -2.8203125,
     -5.08203125,
     -6.96875,
     -5.5859375,
     -4.140625,
     -4.28125,
     -4.2421875,
     -3.12890625,
     -3.302734375,
     -2.708984375,
     -6.21484375,
     -7.1796875,
     -3.234375,
     -2.970703125,
     -6.05078125,
     -5.234375,
     -2.61328125,
     -6.46484375,
     -7.01171875,
     -5.453125,
     -5.671875,
     -3.857421875,
     -5.24609375,
     -7.5703125,
     -3.501953125,
     -3.123046875,
     -4.328125,
     -4.01953125,
     -2.63671875,
     -4.125,
     -4.73828125,
     -6.91796875,
     -6.53515625,
     -6.16015625,
     -5.09375,
     -4.51953125,
     -3.27734375,
     -1.2900390625,
     -2.685546875,
     -1.703125,
     -2.009765625,
     -1.9345703125,
     -1.234375,
     -3.111328125,
     0.33740234375,
     -1.306640625,
     -1.62109375,
     -1.0859375,
     -1.7958984375,
     -2.455078125,
     -1.759765625,
     -3.5234375,
     -1.7412109375,
     -5.19140625,
     -2.677734375,
     -3.50390625,
     -2.302734375,
     -1.9892578125,
     -3.646484375,
     -1.50390625,
     -3.34765625,
     -3.25,
     -3.390625,
     -3.314453125,
     -4.24609375,
     -4.1796875,
     -2.740234375,
     -2.517578125,
     -3.083984375,
     -2.88671875,
     -2.59765625,
     -2.966796875,
     -1.830078125,
     -3.287109375,
     -2.64453125,
     -2.48828125,
     -2.26953125,
     -3.486328125,
     -2.857421875,
     -1.818359375,
     -1.712890625,
     -1.8583984375,
     -2.205078125,
     -2.171875,
     -1.060546875,
     -1.15625,
     -3.193359375,
     -1.609375,
     -1.78515625,
     -2.83203125,
     -1.5693359375,
     -2.3359375,
     -3.056640625,
     0.1815185546875,
     -2.54296875,
     -1.0068359375,
     -1.263671875,
     -1.0810546875,
     -2.001953125,
     -4.06640625,
     -2.01171875,
     -1.3134765625,
     -2.392578125,
     -1.2353515625,
     -2.029296875,
     -1.7548828125,
     -2.248046875,
     -2.21875,
     -0.97607421875,
     0.471435546875,
     -0.5966796875,
     -0.070556640625,
     0.35888671875,
     0.8896484375,
     -1.2548828125,
     0.2548828125,
     0.259765625,
     0.043853759765625,
     -0.70703125,
     0.14501953125,
     1.3681640625,
     1.3408203125,
     -0.2469482421875,
     0.404052734375,
     -1.0048828125,
     0.288818359375,
     0.6142578125,
     -0.180419921875,
     -0.0248565673828125,
     0.2001953125,
     -0.01141357421875,
     0.4150390625,
     -0.019775390625,
     -1.2705078125,
     2.15625,
     -0.81494140625,
     0.6298828125,
     -0.472900390625,
     0.110107421875,
     0.171630859375,
     -4.46875
    ],
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-22T19:27:05+00:00",
   "generated_at": "2026-09-22T19:27:05+00:00",
   "wall_seconds_contended": 2.810466874972917,
   "kind": "wide",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/fp16-02-wide-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/readout-fp16-02-wide-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/conversion/slot/readout_gate_slot.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/fp16-02-wide-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 6.796192666981369,
   "mpsgraph_scratch_before": {
    "at": "2026-09-22T19:26:59+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   },
   "mpsgraph_scratch_after": {
    "at": "2026-09-22T19:27:06+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   }
  },
  {
   "schema": "coreai-slot-readout-session/1",
   "session": "fp16-03-wide-a1",
   "mode": "fp16",
   "indices": [
    49
   ],
   "pid": 13558,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_fp16.h16c.aimodelc",
   "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
   "started_at": "2026-09-22T19:27:09+00:00",
   "phase": "complete",
   "runtime_calls": 2898,
   "planned_runtime_calls": 2898,
   "result": "PASS",
   "load_wall_seconds_contended": 0.233345584012568,
   "active_row": "r18-slot",
   "active_step": 1448,
   "reset_check": {
    "row": "r18-slot",
    "identical": true,
    "max_abs_diff": 0.0,
    "first_raw_logits": [
     -5.13671875,
     -5.14453125,
     -7.3203125,
     -6.67578125,
     -7.55859375,
     -8.15625,
     -7.15625,
     -8.46875,
     -9.484375,
     -6.359375,
     -9.1953125,
     -9.2578125,
     -7.421875,
     -9.5,
     -9.140625,
     -8.140625,
     -10.2421875,
     -9.453125,
     -7.890625,
     -10.2578125,
     -9.3515625,
     -7.90234375,
     -10.3359375,
     -10.1796875,
     -11.265625,
     -12.0703125,
     -10.484375,
     -11.15625,
     -10.1640625,
     -10.4140625,
     -9.5703125,
     -13.34375,
     -11.4765625,
     -10.3984375,
     -9.8203125,
     -11.671875,
     -11.40625,
     -9.2578125,
     -9.6875,
     -7.7421875,
     -11.140625,
     -11.078125,
     -12.4609375,
     -10.28125,
     -10.3984375,
     -12.0234375,
     -10.0390625,
     -9.9375,
     -10.6484375,
     -10.6953125,
     -13.140625,
     -11.5234375,
     -12.484375,
     -11.125,
     -10.96875,
     -9.3359375,
     -9.453125,
     -8.2890625,
     -9.2734375,
     -7.10546875,
     -14.875,
     -13.765625,
     -15.0,
     -11.109375,
     -11.1484375,
     -18.265625,
     -16.890625,
     -9.046875,
     -17.28125,
     -11.0,
     -16.21875,
     -10.1328125,
     -7.30078125,
     -12.03125,
     -9.1640625,
     -15.0703125,
     -10.078125,
     -14.4296875,
     -8.453125,
     -12.0859375,
     -8.7421875,
     -10.8046875,
     -7.0234375,
     -12.6015625,
     -8.7109375,
     -10.0,
     -8.6171875,
     -9.9453125,
     -9.4375,
     -14.5546875,
     -11.7109375,
     -13.1484375,
     -10.1640625,
     -10.09375,
     -9.7109375,
     -11.546875,
     -9.75,
     -7.05859375,
     -11.921875,
     -9.90625,
     -16.09375,
     -8.671875,
     -8.8359375,
     -10.859375,
     -8.796875,
     -15.875,
     -11.0625,
     -12.3984375,
     -9.890625,
     -14.3515625,
     -8.0703125,
     -14.3515625,
     -14.4765625,
     -15.6484375,
     -10.3359375,
     -6.703125,
     -10.1953125,
     -15.296875,
     -12.9453125,
     -10.171875,
     -8.2421875,
     -7.0234375,
     -8.03125,
     -9.453125,
     -7.29296875,
     -14.96875,
     -15.4921875,
     -9.6484375,
     -8.9296875,
     -16.5,
     -12.1171875,
     -11.7265625,
     -16.265625,
     -16.046875,
     -8.21875,
     -15.328125,
     -10.6953125,
     -10.34375,
     -16.3125,
     -12.625,
     -9.828125,
     -11.6796875,
     -10.046875,
     -9.9765625,
     -9.171875,
     -10.453125,
     -16.265625,
     -14.453125,
     -11.3828125,
     -10.4375,
     -11.71875,
     -10.7109375,
     -10.8515625,
     -10.203125,
     -6.0078125,
     -5.8046875,
     -4.9375,
     -5.40625,
     -7.50390625,
     -9.0390625,
     -6.68359375,
     -6.46484375,
     -6.3125,
     -6.328125,
     -9.5546875,
     -3.634765625,
     -7.07421875,
     -7.25,
     -10.8984375,
     -8.2265625,
     -7.57421875,
     -7.55859375,
     -4.484375,
     -9.2890625,
     -7.44140625,
     -7.515625,
     -11.0,
     -9.734375,
     -11.375,
     -10.1484375,
     -10.9921875,
     -12.890625,
     -10.1328125,
     -10.8046875,
     -11.4921875,
     -9.828125,
     -9.96875,
     -10.1640625,
     -11.078125,
     -11.6953125,
     -11.3984375,
     -9.890625,
     -10.734375,
     -9.5,
     -11.375,
     -10.4453125,
     -11.4140625,
     -9.375,
     -9.8359375,
     -10.203125,
     -9.6484375,
     -10.5703125,
     -8.3984375,
     -11.1328125,
     -9.765625,
     -8.7890625,
     -10.2421875,
     -8.859375,
     -0.71728515625,
     -10.7734375,
     -8.90625,
     -9.8359375,
     -9.4140625,
     -9.140625,
     -9.375,
     -9.125,
     -10.3203125,
     -8.5625,
     -9.046875,
     -5.93359375,
     -7.08203125,
     -0.01508331298828125,
     -1.7861328125,
     -1.0888671875,
     1.55078125,
     -0.8447265625,
     0.137939453125,
     -0.4443359375,
     -0.75390625,
     0.271484375,
     0.87890625,
     1.505859375,
     -2.0078125,
     -0.11822509765625,
     1.19921875,
     0.75390625,
     0.355712890625,
     -0.58544921875,
     0.86767578125,
     0.39208984375,
     -0.014862060546875,
     -1.2626953125,
     -1.3330078125,
     1.1806640625,
     -0.269775390625,
     1.7109375,
     -1.1708984375,
     0.142822265625,
     -1.3623046875,
     -0.2188720703125,
     -1.4833984375,
     1.1357421875,
     -0.2069091796875,
     1.8173828125,
     -1.251953125,
     -4.75
    ],
    "final_raw_logits": [
     -5.13671875,
     -5.14453125,
     -7.3203125,
     -6.67578125,
     -7.55859375,
     -8.15625,
     -7.15625,
     -8.46875,
     -9.484375,
     -6.359375,
     -9.1953125,
     -9.2578125,
     -7.421875,
     -9.5,
     -9.140625,
     -8.140625,
     -10.2421875,
     -9.453125,
     -7.890625,
     -10.2578125,
     -9.3515625,
     -7.90234375,
     -10.3359375,
     -10.1796875,
     -11.265625,
     -12.0703125,
     -10.484375,
     -11.15625,
     -10.1640625,
     -10.4140625,
     -9.5703125,
     -13.34375,
     -11.4765625,
     -10.3984375,
     -9.8203125,
     -11.671875,
     -11.40625,
     -9.2578125,
     -9.6875,
     -7.7421875,
     -11.140625,
     -11.078125,
     -12.4609375,
     -10.28125,
     -10.3984375,
     -12.0234375,
     -10.0390625,
     -9.9375,
     -10.6484375,
     -10.6953125,
     -13.140625,
     -11.5234375,
     -12.484375,
     -11.125,
     -10.96875,
     -9.3359375,
     -9.453125,
     -8.2890625,
     -9.2734375,
     -7.10546875,
     -14.875,
     -13.765625,
     -15.0,
     -11.109375,
     -11.1484375,
     -18.265625,
     -16.890625,
     -9.046875,
     -17.28125,
     -11.0,
     -16.21875,
     -10.1328125,
     -7.30078125,
     -12.03125,
     -9.1640625,
     -15.0703125,
     -10.078125,
     -14.4296875,
     -8.453125,
     -12.0859375,
     -8.7421875,
     -10.8046875,
     -7.0234375,
     -12.6015625,
     -8.7109375,
     -10.0,
     -8.6171875,
     -9.9453125,
     -9.4375,
     -14.5546875,
     -11.7109375,
     -13.1484375,
     -10.1640625,
     -10.09375,
     -9.7109375,
     -11.546875,
     -9.75,
     -7.05859375,
     -11.921875,
     -9.90625,
     -16.09375,
     -8.671875,
     -8.8359375,
     -10.859375,
     -8.796875,
     -15.875,
     -11.0625,
     -12.3984375,
     -9.890625,
     -14.3515625,
     -8.0703125,
     -14.3515625,
     -14.4765625,
     -15.6484375,
     -10.3359375,
     -6.703125,
     -10.1953125,
     -15.296875,
     -12.9453125,
     -10.171875,
     -8.2421875,
     -7.0234375,
     -8.03125,
     -9.453125,
     -7.29296875,
     -14.96875,
     -15.4921875,
     -9.6484375,
     -8.9296875,
     -16.5,
     -12.1171875,
     -11.7265625,
     -16.265625,
     -16.046875,
     -8.21875,
     -15.328125,
     -10.6953125,
     -10.34375,
     -16.3125,
     -12.625,
     -9.828125,
     -11.6796875,
     -10.046875,
     -9.9765625,
     -9.171875,
     -10.453125,
     -16.265625,
     -14.453125,
     -11.3828125,
     -10.4375,
     -11.71875,
     -10.7109375,
     -10.8515625,
     -10.203125,
     -6.0078125,
     -5.8046875,
     -4.9375,
     -5.40625,
     -7.50390625,
     -9.0390625,
     -6.68359375,
     -6.46484375,
     -6.3125,
     -6.328125,
     -9.5546875,
     -3.634765625,
     -7.07421875,
     -7.25,
     -10.8984375,
     -8.2265625,
     -7.57421875,
     -7.55859375,
     -4.484375,
     -9.2890625,
     -7.44140625,
     -7.515625,
     -11.0,
     -9.734375,
     -11.375,
     -10.1484375,
     -10.9921875,
     -12.890625,
     -10.1328125,
     -10.8046875,
     -11.4921875,
     -9.828125,
     -9.96875,
     -10.1640625,
     -11.078125,
     -11.6953125,
     -11.3984375,
     -9.890625,
     -10.734375,
     -9.5,
     -11.375,
     -10.4453125,
     -11.4140625,
     -9.375,
     -9.8359375,
     -10.203125,
     -9.6484375,
     -10.5703125,
     -8.3984375,
     -11.1328125,
     -9.765625,
     -8.7890625,
     -10.2421875,
     -8.859375,
     -0.71728515625,
     -10.7734375,
     -8.90625,
     -9.8359375,
     -9.4140625,
     -9.140625,
     -9.375,
     -9.125,
     -10.3203125,
     -8.5625,
     -9.046875,
     -5.93359375,
     -7.08203125,
     -0.01508331298828125,
     -1.7861328125,
     -1.0888671875,
     1.55078125,
     -0.8447265625,
     0.137939453125,
     -0.4443359375,
     -0.75390625,
     0.271484375,
     0.87890625,
     1.505859375,
     -2.0078125,
     -0.11822509765625,
     1.19921875,
     0.75390625,
     0.355712890625,
     -0.58544921875,
     0.86767578125,
     0.39208984375,
     -0.014862060546875,
     -1.2626953125,
     -1.3330078125,
     1.1806640625,
     -0.269775390625,
     1.7109375,
     -1.1708984375,
     0.142822265625,
     -1.3623046875,
     -0.2188720703125,
     -1.4833984375,
     1.1357421875,
     -0.2069091796875,
     1.8173828125,
     -1.251953125,
     -4.75
    ],
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-22T19:27:26+00:00",
   "generated_at": "2026-09-22T19:27:26+00:00",
   "wall_seconds_contended": 16.521090624970384,
   "kind": "wide",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/fp16-03-wide-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/readout-fp16-03-wide-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/conversion/slot/readout_gate_slot.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/fp16-03-wide-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 20.463113917037845,
   "mpsgraph_scratch_before": {
    "at": "2026-09-22T19:27:06+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   },
   "mpsgraph_scratch_after": {
    "at": "2026-09-22T19:27:26+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   }
  },
  {
   "schema": "coreai-slot-readout-session/1",
   "session": "fp16-04-global_reset-a1",
   "mode": "fp16",
   "indices": [
    0
   ],
   "pid": 14408,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_fp16.h16c.aimodelc",
   "config_path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/hf/hub/models--iapp--OpenThai-SystemOne/blobs/d03bf878f6563dd505c0a739ce9b7cc1d3bed63b",
   "started_at": "2026-09-22T19:27:30+00:00",
   "phase": "complete",
   "runtime_calls": 226,
   "planned_runtime_calls": 226,
   "result": "PASS",
   "load_wall_seconds_contended": 0.2361370410071686,
   "active_row": "r01-route",
   "active_step": 112,
   "reset_check": {
    "row": "r01-route",
    "identical": true,
    "max_abs_diff": 0.0,
    "first_raw_logits": [
     0.93798828125,
     -4.04296875,
     -1.77734375,
     -3.1484375,
     -3.025390625,
     -3.1171875,
     -3.091796875,
     -3.0546875,
     -4.25,
     -2.951171875,
     -3.03125,
     -2.72265625,
     -4.34375,
     -3.388671875,
     -3.658203125,
     -3.583984375,
     -3.96484375,
     -4.8984375,
     -3.572265625,
     -3.408203125,
     -3.673828125,
     -2.38671875,
     -5.4921875,
     -5.2109375,
     -3.927734375,
     -4.7734375,
     -3.419921875,
     -4.7265625,
     -4.140625,
     -5.67578125,
     -4.296875,
     -4.5234375,
     -5.80859375,
     -3.939453125,
     -3.271484375,
     -4.9296875,
     -5.73046875,
     -6.21484375,
     -4.6953125,
     -3.015625,
     -5.89453125,
     -4.79296875,
     -5.14453125,
     -5.94921875,
     -3.826171875,
     -4.9609375,
     -4.34375,
     -4.94921875,
     -5.390625,
     -4.69140625,
     -5.26953125,
     -4.62109375,
     -3.86328125,
     -5.46875,
     -3.400390625,
     -3.7734375,
     -3.9453125,
     -4.4375,
     -4.578125,
     -5.76953125,
     -7.4453125,
     -7.1953125,
     -8.125,
     -5.8125,
     -3.712890625,
     -7.5,
     -8.015625,
     -2.70703125,
     -7.85546875,
     -4.3515625,
     -7.98828125,
     -5.375,
     -2.759765625,
     -5.4140625,
     -3.76171875,
     -6.3671875,
     -4.109375,
     -7.03125,
     -4.02734375,
     -6.0546875,
     -4.65625,
     -4.25390625,
     -3.181640625,
     -4.75,
     -3.55859375,
     -4.8828125,
     -2.650390625,
     -4.10546875,
     -3.326171875,
     -4.47265625,
     -4.01171875,
     -4.15234375,
     -4.30859375,
     -3.6171875,
     -4.76953125,
     -4.1015625,
     -5.0859375,
     -3.162109375,
     -2.6640625,
     -4.7421875,
     -6.1640625,
     -4.94921875,
     -3.408203125,
     -4.33984375,
     -4.77734375,
     -5.9453125,
     -3.4765625,
     -5.23046875,
     -4.12890625,
     -6.015625,
     -4.171875,
     -4.95703125,
     -5.01953125,
     -5.78125,
     -4.6953125,
     -3.5,
     -4.77734375,
     -5.984375,
     -4.2578125,
     -3.68359375,
     -3.125,
     -4.28515625,
     -2.890625,
     -3.681640625,
     -2.197265625,
     -5.09765625,
     -5.828125,
     -2.65625,
     -2.80078125,
     -5.77734375,
     -5.67578125,
     -3.236328125,
     -5.8125,
     -6.62890625,
     -5.34765625,
     -4.94140625,
     -3.755859375,
     -4.33203125,
     -6.3515625,
     -2.697265625,
     -3.625,
     -4.32421875,
     -3.037109375,
     -2.794921875,
     -4.24609375,
     -4.91796875,
     -5.87890625,
     -5.78515625,
     -5.21484375,
     -4.51171875,
     -4.5546875,
     -2.853515625,
     -1.0400390625,
     -3.15234375,
     -1.9384765625,
     -1.4453125,
     -1.5673828125,
     -1.51953125,
     -2.634765625,
     -0.86865234375,
     -1.8623046875,
     -1.7509765625,
     -1.4150390625,
     -1.34765625,
     -2.693359375,
     -1.841796875,
     -2.9765625,
     -2.271484375,
     -3.876953125,
     -2.115234375,
     -2.2109375,
     -2.541015625,
     -1.865234375,
     -2.91796875,
     -0.94873046875,
     -1.951171875,
     -3.669921875,
     -3.4140625,
     -3.232421875,
     -2.56640625,
     -3.90625,
     -2.49609375,
     -3.078125,
     -2.763671875,
     -3.390625,
     -2.501953125,
     -2.0,
     -2.05078125,
     -2.541015625,
     -2.658203125,
     -1.2587890625,
     -2.01171875,
     -2.76953125,
     -1.8408203125,
     -2.048828125,
     -2.10546875,
     -2.076171875,
     -2.02734375,
     -2.1171875,
     -1.134765625,
     -1.3955078125,
     -2.607421875,
     -1.2646484375,
     -2.27734375,
     -2.55859375,
     -1.7783203125,
     -2.203125,
     -2.5,
     0.07110595703125,
     -2.3203125,
     -1.7548828125,
     -1.232421875,
     -0.10736083984375,
     -1.1083984375,
     -3.306640625,
     -1.12890625,
     -2.017578125,
     -2.140625,
     -1.548828125,
     -2.09765625,
     -1.2158203125,
     -1.3671875,
     -1.0185546875,
     -0.8388671875,
     -0.0960693359375,
     -0.44384765625,
     -0.76806640625,
     0.890625,
     0.74658203125,
     -0.3798828125,
     -0.0428466796875,
     -0.452392578125,
     0.1131591796875,
     -0.91845703125,
     0.08892822265625,
     1.0849609375,
     0.904296875,
     -0.081298828125,
     -0.1031494140625,
     -0.72021484375,
     0.312255859375,
     0.53759765625,
     -0.7060546875,
     0.048126220703125,
     -0.137939453125,
     0.408935546875,
     0.78076171875,
     -0.75927734375,
     -0.13330078125,
     2.06640625,
     -0.72412109375,
     -0.288818359375,
     -0.50244140625,
     -0.0106964111328125,
     -0.3037109375,
     -2.74609375
    ],
    "final_raw_logits": [
     0.93798828125,
     -4.04296875,
     -1.77734375,
     -3.1484375,
     -3.025390625,
     -3.1171875,
     -3.091796875,
     -3.0546875,
     -4.25,
     -2.951171875,
     -3.03125,
     -2.72265625,
     -4.34375,
     -3.388671875,
     -3.658203125,
     -3.583984375,
     -3.96484375,
     -4.8984375,
     -3.572265625,
     -3.408203125,
     -3.673828125,
     -2.38671875,
     -5.4921875,
     -5.2109375,
     -3.927734375,
     -4.7734375,
     -3.419921875,
     -4.7265625,
     -4.140625,
     -5.67578125,
     -4.296875,
     -4.5234375,
     -5.80859375,
     -3.939453125,
     -3.271484375,
     -4.9296875,
     -5.73046875,
     -6.21484375,
     -4.6953125,
     -3.015625,
     -5.89453125,
     -4.79296875,
     -5.14453125,
     -5.94921875,
     -3.826171875,
     -4.9609375,
     -4.34375,
     -4.94921875,
     -5.390625,
     -4.69140625,
     -5.26953125,
     -4.62109375,
     -3.86328125,
     -5.46875,
     -3.400390625,
     -3.7734375,
     -3.9453125,
     -4.4375,
     -4.578125,
     -5.76953125,
     -7.4453125,
     -7.1953125,
     -8.125,
     -5.8125,
     -3.712890625,
     -7.5,
     -8.015625,
     -2.70703125,
     -7.85546875,
     -4.3515625,
     -7.98828125,
     -5.375,
     -2.759765625,
     -5.4140625,
     -3.76171875,
     -6.3671875,
     -4.109375,
     -7.03125,
     -4.02734375,
     -6.0546875,
     -4.65625,
     -4.25390625,
     -3.181640625,
     -4.75,
     -3.55859375,
     -4.8828125,
     -2.650390625,
     -4.10546875,
     -3.326171875,
     -4.47265625,
     -4.01171875,
     -4.15234375,
     -4.30859375,
     -3.6171875,
     -4.76953125,
     -4.1015625,
     -5.0859375,
     -3.162109375,
     -2.6640625,
     -4.7421875,
     -6.1640625,
     -4.94921875,
     -3.408203125,
     -4.33984375,
     -4.77734375,
     -5.9453125,
     -3.4765625,
     -5.23046875,
     -4.12890625,
     -6.015625,
     -4.171875,
     -4.95703125,
     -5.01953125,
     -5.78125,
     -4.6953125,
     -3.5,
     -4.77734375,
     -5.984375,
     -4.2578125,
     -3.68359375,
     -3.125,
     -4.28515625,
     -2.890625,
     -3.681640625,
     -2.197265625,
     -5.09765625,
     -5.828125,
     -2.65625,
     -2.80078125,
     -5.77734375,
     -5.67578125,
     -3.236328125,
     -5.8125,
     -6.62890625,
     -5.34765625,
     -4.94140625,
     -3.755859375,
     -4.33203125,
     -6.3515625,
     -2.697265625,
     -3.625,
     -4.32421875,
     -3.037109375,
     -2.794921875,
     -4.24609375,
     -4.91796875,
     -5.87890625,
     -5.78515625,
     -5.21484375,
     -4.51171875,
     -4.5546875,
     -2.853515625,
     -1.0400390625,
     -3.15234375,
     -1.9384765625,
     -1.4453125,
     -1.5673828125,
     -1.51953125,
     -2.634765625,
     -0.86865234375,
     -1.8623046875,
     -1.7509765625,
     -1.4150390625,
     -1.34765625,
     -2.693359375,
     -1.841796875,
     -2.9765625,
     -2.271484375,
     -3.876953125,
     -2.115234375,
     -2.2109375,
     -2.541015625,
     -1.865234375,
     -2.91796875,
     -0.94873046875,
     -1.951171875,
     -3.669921875,
     -3.4140625,
     -3.232421875,
     -2.56640625,
     -3.90625,
     -2.49609375,
     -3.078125,
     -2.763671875,
     -3.390625,
     -2.501953125,
     -2.0,
     -2.05078125,
     -2.541015625,
     -2.658203125,
     -1.2587890625,
     -2.01171875,
     -2.76953125,
     -1.8408203125,
     -2.048828125,
     -2.10546875,
     -2.076171875,
     -2.02734375,
     -2.1171875,
     -1.134765625,
     -1.3955078125,
     -2.607421875,
     -1.2646484375,
     -2.27734375,
     -2.55859375,
     -1.7783203125,
     -2.203125,
     -2.5,
     0.07110595703125,
     -2.3203125,
     -1.7548828125,
     -1.232421875,
     -0.10736083984375,
     -1.1083984375,
     -3.306640625,
     -1.12890625,
     -2.017578125,
     -2.140625,
     -1.548828125,
     -2.09765625,
     -1.2158203125,
     -1.3671875,
     -1.0185546875,
     -0.8388671875,
     -0.0960693359375,
     -0.44384765625,
     -0.76806640625,
     0.890625,
     0.74658203125,
     -0.3798828125,
     -0.0428466796875,
     -0.452392578125,
     0.1131591796875,
     -0.91845703125,
     0.08892822265625,
     1.0849609375,
     0.904296875,
     -0.081298828125,
     -0.1031494140625,
     -0.72021484375,
     0.312255859375,
     0.53759765625,
     -0.7060546875,
     0.048126220703125,
     -0.137939453125,
     0.408935546875,
     0.78076171875,
     -0.75927734375,
     -0.13330078125,
     2.06640625,
     -0.72412109375,
     -0.288818359375,
     -0.50244140625,
     -0.0106964111328125,
     -0.3037109375,
     -2.74609375
    ],
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-22T19:27:32+00:00",
   "generated_at": "2026-09-22T19:27:32+00:00",
   "wall_seconds_contended": 1.6968784159980714,
   "kind": "global_reset",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/fp16-04-global_reset-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/readout-fp16-04-global_reset-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/zoo/conversion/slot/readout_gate_slot.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/results/round2/slot-gates/readout-fp16/fp16-04-global_reset-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 5.796681415988132,
   "mpsgraph_scratch_before": {
    "at": "2026-09-22T19:27:26+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   },
   "mpsgraph_scratch_after": {
    "at": "2026-09-22T19:27:32+00:00",
    "glob_patterns": [
     "/private/var/folders/*/*/T/com.apple.MetalPerformanceShadersGraph",
     "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/cache/tmp/com.apple.MetalPerformanceShadersGraph"
    ],
    "paths": [
     {
      "path": "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph",
      "command": [
       "du",
       "-sh",
       "/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph"
      ],
      "stdout": "  0B\t/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph\n",
      "stderr": "",
      "returncode": 0
     }
    ],
    "note": "Read-only; nothing deleted"
   }
  }
 ],
 "result": "PASS",
 "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_fp16.h16c.aimodelc",
 "aot": {
  "source": {
   "path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/openthai_systemone_decode_fp16/openthai_systemone_decode_fp16.aimodel",
   "bytes": 160,
   "mtime_ns": 1790102168038081977
  },
  "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/openthai-systemone-coreai/exports/aotc/openthai_systemone_decode_fp16.h16c.aimodelc",
  "result": "PASS",
  "reused": true,
  "wall_seconds": 0.0,
  "note": "Existing explicitly selected AOT asset reused; no GPU JIT."
 },
 "aot_seconds": 0.0,
 "summary": {
  "rows": 50,
  "expected_rows": 50,
  "full_vocab_argmax_is_label": 43,
  "argmax_agreement": 50,
  "margin_ge_0_02_rows": 49,
  "margin_ge_0_02_argmax_agreement": 49,
  "near_ties": [
   {
    "id": "r18-slot",
    "oracle_margin": 0.009739205241203308,
    "oracle_argmax": 253,
    "option_argmax": 253,
    "p": [
     0.00015095308481249958,
     0.00014984306471887976,
     1.9184879420208745e-05,
     3.526930231601e-05,
     1.531782436359208e-05,
     8.709457688382827e-06,
     2.240115645690821e-05,
     6.483043762273155e-06,
     2.4836417651386e-06,
     4.755667396239005e-05,
     3.2635123261570698e-06,
     3.0763999347982462e-06,
     1.7429692888981663e-05,
     2.4472503810102353e-06,
     3.436547558521852e-06,
     8.838970643409993e-06,
     1.2138752936152741e-06,
     2.558057076385012e-06,
     1.1193642421858385e-05,
     1.1960888741668896e-06,
     2.815653033394483e-06,
     1.107040952774696e-05,
     1.1109907518402906e-06,
     1.2877052313342574e-06,
     4.616138369328837e-07,
     2.1584027365406655e-07,
     9.656275778979762e-07,
     5.118627086631022e-07,
     1.3068540738458978e-06,
     1.0319470220565563e-06,
     2.2899744180904236e-06,
     6.481379699607714e-08,
     3.782122632856044e-07,
     1.04729247141222e-06,
     1.8082600945490412e-06,
     3.144872096072504e-07,
     4.04187943559009e-07,
     3.0763999347982462e-06,
     2.0499867332546273e-06,
     1.2878706002084073e-05,
     5.194743266656587e-07,
     5.510697178578994e-07,
     1.4923368496511102e-07,
     1.169896790997882e-06,
     1.04729247141222e-06,
     2.256131068634204e-07,
     1.4706589581692242e-06,
     1.6187558458113926e-06,
     8.269861950793711e-07,
     7.911638135738031e-07,
     7.85245362067144e-08,
     3.6182933627060265e-07,
     1.4596575681480317e-07,
     5.27199176758586e-07,
     6.110556682870083e-07,
     2.8575232136063278e-06,
     2.558057076385012e-06,
     7.682471732550766e-06,
     3.0313226488942746e-06,
     2.3501994292018935e-05,
     1.5255519514312255e-08,
     4.350915361328589e-08,
     1.355633116872923e-08,
     5.35038850557612e-07,
     5.156545057616313e-07,
     6.19903073317829e-10,
     2.272260823943384e-09,
     3.754793397092726e-06,
     1.5710607259578069e-09,
     5.932798785579507e-07,
     4.286613730641875e-09,
     1.3460102081808145e-06,
     1.9542139853001572e-05,
     2.2395408905140357e-07,
     3.36129346578673e-06,
     1.2685116068666957e-08,
     1.4173772342473967e-06,
     2.3234312251929623e-08,
     6.579449291166384e-06,
     2.1267766214805306e-07,
     5.007180334359873e-06,
     7.134970019251341e-07,
     2.5395724151167087e-05,
     1.3066863857602584e-07,
     5.1572060328908265e-06,
     1.5259450947269215e-06,
     5.6347957979596686e-06,
     1.606852606528264e-06,
     2.596096464912989e-06,
     2.0646428566806208e-08,
     3.030933726222429e-07,
     7.794712786335367e-08,
     1.3068540738458978e-06,
     1.3966090364192496e-06,
     2.0050961211381946e-06,
     3.539059321155946e-07,
     1.93245182344981e-06,
     2.456611764500849e-05,
     2.483323271462723e-07,
     1.667257151893864e-06,
     4.823909716122898e-09,
     5.351074833015446e-06,
     4.582786004903028e-06,
     6.77571335927496e-07,
     4.755062036565505e-06,
     5.931270141701361e-09,
     5.592642651208735e-07,
     1.5831035682367656e-07,
     1.6920498637773562e-06,
     2.5013983773192194e-08,
     9.446032890991773e-06,
     2.5013983773192194e-08,
     2.2227876428360105e-08,
     7.346863117874136e-09,
     1.1109907518402906e-06,
     3.436988117755391e-05,
     1.2688371953117894e-06,
     1.0240948533635219e-08,
     9.443608917081292e-08,
     1.297244352826965e-06,
     8.030317985685542e-06,
     2.5395724151167087e-05,
     9.801125997910276e-06,
     2.558057076385012e-06,
     1.968690412468277e-05,
     1.3962506706377553e-08,
     8.515458560509614e-09,
     2.1270493562042248e-06,
     4.194359007669846e-06,
     3.2864224674966636e-09,
     2.0649076759582385e-07,
     2.986523099934857e-07,
     4.100931594308577e-09,
     5.042327000381874e-09,
     8.21010416984791e-06,
     9.943035728099403e-09,
     7.911638135738031e-07,
     1.1028211019947776e-06,
     3.923292357654873e-09,
     1.278072545574105e-07,
     1.7949635093827965e-06,
     3.12174705641155e-07,
     1.4598448387914686e-06,
     1.5601085578964557e-06,
     3.336577037771349e-06,
     9.945596275429125e-07,
     4.100931594308577e-09,
     2.2725522796918085e-08,
     4.1323704635942704e-07,
     1.0093492619489552e-06,
     3.0086465585554834e-07,
     7.795712804181676e-07,
     6.825906666563242e-07,
     1.259507030226814e-06,
     6.629035488003865e-05,
     8.031344623304904e-05,
     0.0001822121994337067,
     0.00011701970652211457,
     1.6129992218338884e-05,
     3.7826080188096967e-06,
     3.500995444483124e-05,
     4.30466971010901e-05,
     4.970994632458314e-05,
     4.8981568397721276e-05,
     2.32402749134053e-06,
     0.0006238245405256748,
     2.4206161469919607e-05,
     2.0502498955465853e-05,
     6.530230507451051e-07,
     8.149731911544222e-06,
     1.5093378351593856e-05,
     1.531782436359208e-05,
     0.00027956647682003677,
     2.986906110891141e-06,
     1.7111049601226114e-05,
     1.5952400644891895e-05,
     5.932798785579507e-07,
     1.9611882180470275e-06,
     4.162982065736287e-07,
     1.3262875881991931e-06,
     5.976746706437552e-07,
     9.944321277544077e-08,
     1.3460102081808145e-06,
     7.134970019251341e-07,
     3.7267048469402653e-07,
     1.7949635093827965e-06,
     1.5716655070718843e-06,
     1.3068540738458978e-06,
     5.510697178578994e-07,
     3.076005441471352e-07,
     4.0718205696066434e-07,
     1.6920498637773562e-06,
     7.625000648658897e-07,
     2.4472503810102353e-06,
     4.162982065736287e-07,
     1.0019271030614618e-06,
     4.0121580013874336e-07,
     2.753995659077191e-06,
     1.7817645812101546e-06,
     1.259507030226814e-06,
     2.1270493562042248e-06,
     8.90330682068452e-07,
     6.928300081199268e-06,
     5.233225124356977e-07,
     1.904136297525838e-06,
     4.7902867663651705e-06,
     1.2138752936152741e-06,
     4.482427812035894e-06,
     0.009818477556109428,
     7.348749022639822e-07,
     4.2882638808805496e-06,
     1.7817645812101546e-06,
     2.6542188606981654e-06,
     3.436547558521852e-06,
     2.753995659077191e-06,
     3.4876504741987446e-06,
     1.1275116094111581e-06,
     5.933559350523865e-06,
     3.754793397092726e-06,
     7.110509613994509e-05,
     2.4028167899814434e-05,
     0.019060881808400154,
     0.0035769930109381676,
     0.006911808624863625,
     0.083672896027565,
     0.008704775013029575,
     0.022025465965270996,
     0.012706602923572063,
     0.009484605863690376,
     0.024987095966935158,
     0.04435354843735695,
     0.08019628375768661,
     0.002901132218539715,
     0.0172912385314703,
     0.06002691015601158,
     0.03941335901618004,
     0.027056585997343063,
     0.011120740324258804,
     0.043885473161935806,
     0.028002556413412094,
     0.01906486600637436,
     0.005865074694156647,
     0.005488147959113121,
     0.05898388475179672,
     0.01498470176011324,
     0.09734050929546356,
     0.00639640586450696,
     0.022127298638224602,
     0.005338337738066912,
     0.01572289876639843,
     0.0047612800262868404,
     0.05653310939669609,
     0.0159015990793705,
     0.10763806104660034,
     0.005924900062382221
    ],
    "p_oracle": [
     0.0001469101116526872,
     0.0001461399078834802,
     1.8727199858403765e-05,
     3.445339825702831e-05,
     1.491959483246319e-05,
     8.525082193955313e-06,
     2.203234726039227e-05,
     6.327398750727298e-06,
     2.4345908968825825e-06,
     4.6558663598261774e-05,
     3.151977125526173e-06,
     2.988307187479222e-06,
     1.7149484847323038e-05,
     2.3426230200129794e-06,
     3.3076789804908913e-06,
     8.558708032069262e-06,
     1.175259626506886e-06,
     2.548649490563548e-06,
     1.0875735824811272e-05,
     1.174447334051365e-06,
     2.7018331820727326e-06,
     1.073516978067346e-05,
     1.0747947953859693e-06,
     1.2419112636052887e-06,
     4.439075667050929e-07,
     2.042345670361101e-07,
     9.404049592376396e-07,
     4.902366299575078e-07,
     1.2479507631724118e-06,
     9.923292054736521e-07,
     2.2280037228483707e-06,
     5.986074569364064e-08,
     3.658147988971905e-07,
     1.0361229669797467e-06,
     1.7353581824863795e-06,
     3.016077982920251e-07,
     3.907217376308836e-07,
     2.925652097474085e-06,
     1.966859144886257e-06,
     1.2392310054565314e-05,
     5.118952799421095e-07,
     5.359339070309943e-07,
     1.4427850203446724e-07,
     1.123934225688572e-06,
     1.0238458116873517e-06,
     2.1720221354826208e-07,
     1.4275401554186828e-06,
     1.5700845779065276e-06,
     8.077286679508688e-07,
     7.595568831675337e-07,
     7.496300469256312e-08,
     3.4219951317027153e-07,
     1.4025690120433865e-07,
     5.061549472884508e-07,
     5.872788619853964e-07,
     2.820014060489484e-06,
     2.5274971449107397e-06,
     7.469083357136697e-06,
     2.887191612899187e-06,
     2.2984444512985647e-05,
     1.4499901723752373e-08,
     4.224759564408487e-08,
     1.3127833931037003e-08,
     5.267225446914381e-07,
     5.032041485719674e-07,
     5.929003399351984e-10,
     2.150327471639457e-09,
     3.6878504943160806e-06,
     1.5137613385007853e-09,
     5.771068458670925e-07,
     4.05874622799729e-09,
     1.283262236029259e-06,
     1.8895223547588103e-05,
     2.1562215124504291e-07,
     3.2647783427819377e-06,
     1.216329259534632e-08,
     1.3845196917827707e-06,
     2.249413455501781e-08,
     6.356727226375369e-06,
     2.0436063152828865e-07,
     4.80034123029327e-06,
     6.884560548314766e-07,
     2.4657274479977787e-05,
     1.261227424720346e-07,
     5.058232545707142e-06,
     1.4855725112283835e-06,
     5.549032721319236e-06,
     1.5243693951561e-06,
     2.5501326490484644e-06,
     1.9503026749134733e-08,
     2.894741442105442e-07,
     7.430752901882443e-08,
     1.2425402928784024e-06,
     1.359587031402043e-06,
     1.943861207109876e-06,
     3.3811633670666197e-07,
     1.8900060467785806e-06,
     2.3787959435139783e-05,
     2.389311362094304e-07,
     1.6118586927404976e-06,
     4.5472798859691466e-09,
     5.195611720409943e-06,
     4.406738753459649e-06,
     6.553700018230302e-07,
     4.563604306895286e-06,
     5.59060486793328e-09,
     5.400765417107323e-07,
     1.514987673090218e-07,
     1.6380289480366628e-06,
     2.4197854386898143e-08,
     9.32275997911347e-06,
     2.393656828303392e-08,
     2.1264302318968475e-08,
     6.9980532479974045e-09,
     1.081376012734836e-06,
     3.306658618384972e-05,
     1.222553464685916e-06,
     9.822396229708374e-09,
     9.009193746578603e-08,
     1.2590995766004198e-06,
     7.929240382509306e-06,
     2.503086216165684e-05,
     9.561872502672486e-06,
     2.482203854015097e-06,
     1.9406297724344768e-05,
     1.3104779483796847e-08,
     8.068131940319745e-09,
     2.05597734748153e-06,
     4.0540408008382656e-06,
     3.169926543478141e-09,
     1.9746653379115742e-07,
     2.882631235934241e-07,
     3.9420209319018795e-09,
     4.785592810918615e-09,
     8.075558071141131e-06,
     9.4671372963262e-09,
     7.656602178940375e-07,
     1.0820309626069502e-06,
     3.729021536003074e-09,
     1.2206049859742052e-07,
     1.735182763695775e-06,
     3.0206345513761335e-07,
     1.4078510730541893e-06,
     1.526253186057147e-06,
     3.32214199261216e-06,
     9.722509730636375e-07,
     3.903593004395134e-09,
     2.2041644953674222e-08,
     4.0664875200491224e-07,
     9.705481716082431e-07,
     2.8843609811701754e-07,
     7.440521017088031e-07,
     6.660162625848898e-07,
     1.2110650686736335e-06,
     6.550806574523449e-05,
     7.979811925906688e-05,
     0.0001818382297642529,
     0.00011510254262248054,
     1.574801717652008e-05,
     3.7079730645928066e-06,
     3.386830576346256e-05,
     4.226343662594445e-05,
     4.857313251704909e-05,
     4.7070567234186456e-05,
     2.2569599877897417e-06,
     0.0006120369071140885,
     2.3661097657168284e-05,
     2.0005791157018393e-05,
     6.244418386813777e-07,
     7.96031963545829e-06,
     1.4662005014542956e-05,
     1.4956691302359104e-05,
     0.0002772198058664799,
     2.891156555051566e-06,
     1.6640678950352594e-05,
     1.5442494259332307e-05,
     5.737019250773301e-07,
     1.9168603557773167e-06,
     3.9845309629527037e-07,
     1.2928065871165018e-06,
     5.787465511275514e-07,
     9.430754488448656e-08,
     1.3109068959238357e-06,
     6.849967917332833e-07,
     3.59871819455293e-07,
     1.7630660522627295e-06,
     1.5343573522841325e-06,
     1.2713866226476966e-06,
     5.288820261739602e-07,
     2.9492520070562023e-07,
     3.9059952428033284e-07,
     1.6423742863480584e-06,
     7.348767780968046e-07,
     2.3556704036309384e-06,
     4.0035379811342864e-07,
     9.642487839300884e-07,
     3.915128559128789e-07,
     2.6290808818885125e-06,
     1.7516496200187248e-06,
     1.2177035841887118e-06,
     2.065924718408496e-06,
     8.670426154822053e-07,
     6.833991392340977e-06,
     4.97766279750067e-07,
     1.818960640775913e-06,
     4.63209062218084e-06,
     1.1824243983937777e-06,
     4.360892944532679e-06,
     0.009755837731063366,
     7.05798640865396e-07,
     4.1220273487851955e-06,
     1.7139896044682246e-06,
     2.5792653559619794e-06,
     3.3125340905826306e-06,
     2.6990644528268604e-06,
     3.3724088552844478e-06,
     1.0858207133424003e-06,
     5.779492767032934e-06,
     3.6232527236279566e-06,
     6.950746319489554e-05,
     2.359280188102275e-05,
     0.018962303176522255,
     0.0035667235497385263,
     0.006895596627146006,
     0.08383864909410477,
     0.008625222370028496,
     0.022189911454916,
     0.012730501592159271,
     0.009463458321988583,
     0.024959232658147812,
     0.0444105863571167,
     0.08021362125873566,
     0.0028592399321496487,
     0.017242614179849625,
     0.060589153319597244,
     0.039358872920274734,
     0.02689761482179165,
     0.011007904075086117,
     0.044373709708452225,
     0.02796688862144947,
     0.0190079715102911,
     0.005807948764413595,
     0.005553585942834616,
     0.05795270577073097,
     0.014917745254933834,
     0.0977199524641037,
     0.006410952657461166,
     0.022017937153577805,
     0.005312574561685324,
     0.015614868141710758,
     0.004747352562844753,
     0.057182542979717255,
     0.01578008383512497,
     0.107459157705307,
     0.005929228384047747
    ]
   }
  ],
  "max_abs_delta_p": 0.005058646202087402,
  "mean_of_row_mean_abs_delta_p": 0.00022480956720861644,
  "max_abs_delta_abstain": 0.01648791879415512,
  "max_abs_delta_raw_logits": 0.10010689496994019,
  "finite_all": true,
  "nonconstant_all": true,
  "reset_identical": true,
  "session_resets_identical": true,
  "provisional_fp16_max_delta_p_expectation": 0.02,
  "provisional_expectation_exceeded": false,
  "worst_five": [
   {
    "id": "r05-task",
    "request_id": "r05",
    "question_id": "task",
    "type": "choice",
    "qtype_index": 0,
    "kind": "slot",
    "tokens": 99,
    "nopts": 3,
    "temperature": 1.058533787727356,
    "raw_logits": [
     1.18359375,
     -1.4716796875,
     -4.47265625,
     -3.32421875,
     -3.16015625,
     -3.111328125,
     -2.943359375,
     -4.32421875,
     -5.00390625,
     -3.47265625,
     -2.908203125,
     -3.1015625,
     -5.19921875,
     -3.376953125,
     -3.912109375,
     -3.89453125,
     -4.2421875,
     -4.73046875,
     -3.396484375,
     -3.80859375,
     -3.5390625,
     -2.41796875,
     -5.375,
     -5.8515625,
     -4.20703125,
     -5.3984375,
     -2.912109375,
     -5.55859375,
     -4.46875,
     -6.36328125,
     -4.62109375,
     -4.34765625,
     -6.53515625,
     -4.3828125,
     -3.740234375,
     -5.33203125,
     -5.703125,
     -5.75,
     -4.31640625,
     -3.107421875,
     -6.1875,
     -5.22265625,
     -5.4765625,
     -5.5,
     -3.884765625,
     -5.0703125,
     -4.30078125,
     -4.94921875,
     -5.390625,
     -4.6171875,
     -5.5078125,
     -5.0,
     -3.7578125,
     -5.21875,
     -3.310546875,
     -3.703125,
     -4.0,
     -4.17578125,
     -4.66796875,
     -5.55859375,
     -7.578125,
     -7.0703125,
     -8.2265625,
     -6.27734375,
     -3.841796875,
     -7.5390625,
     -7.89453125,
     -2.837890625,
     -8.046875,
     -4.171875,
     -8.015625,
     -5.4296875,
     -2.703125,
     -5.5859375,
     -4.046875,
     -6.3125,
     -4.234375,
     -7.1171875,
     -3.96875,
     -6.03125,
     -4.265625,
     -4.15625,
     -2.90625,
     -4.7265625,
     -3.6796875,
     -4.57421875,
     -2.64453125,
     -4.03515625,
     -3.486328125,
     -4.921875,
     -3.916015625,
     -4.66015625,
     -4.46875,
     -3.83203125,
     -4.578125,
     -4.0234375,
     -5.02734375,
     -3.5390625,
     -2.97265625,
     -4.390625,
     -6.21875,
     -4.8359375,
     -3.515625,
     -4.69921875,
     -5.04296875,
     -6.19140625,
     -4.08984375,
     -5.5078125,
     -4.03515625,
     -5.93359375,
     -4.23046875,
     -5.0,
     -5.2421875,
     -6.2578125,
     -4.73046875,
     -3.57421875,
     -4.67578125,
     -6.08203125,
     -4.6796875,
     -3.96484375,
     -3.11328125,
     -4.12109375,
     -2.98828125,
     -3.91796875,
     -2.201171875,
     -5.38671875,
     -5.921875,
     -2.69921875,
     -3.0625,
     -5.91015625,
     -5.66796875,
     -3.716796875,
     -6.046875,
     -6.7109375,
     -5.18359375,
     -5.0234375,
     -3.73828125,
     -4.59375,
     -6.3828125,
     -2.89453125,
     -3.951171875,
     -4.3984375,
     -3.26953125,
     -3.056640625,
     -4.32421875,
     -5.24609375,
     -6.1796875,
     -5.8828125,
     -4.95703125,
     -4.30859375,
     -4.5,
     -2.837890625,
     -1.2744140625,
     -3.25390625,
     -2.08984375,
     -1.787109375,
     -1.3837890625,
     -1.6474609375,
     -2.958984375,
     -1.142578125,
     -2.1171875,
     -1.6982421875,
     -1.578125,
     -1.328125,
     -2.392578125,
     -2.2265625,
     -2.94140625,
     -2.49609375,
     -3.66015625,
     -2.37890625,
     -2.486328125,
     -2.658203125,
     -2.365234375,
     -3.16796875,
     -1.130859375,
     -1.9326171875,
     -3.90234375,
     -3.578125,
     -3.587890625,
     -3.06640625,
     -4.55078125,
     -3.076171875,
     -3.072265625,
     -3.07421875,
     -3.158203125,
     -2.490234375,
     -1.892578125,
     -2.17578125,
     -2.78515625,
     -2.65625,
     -1.5693359375,
     -2.220703125,
     -3.328125,
     -2.001953125,
     -1.8359375,
     -1.908203125,
     -2.552734375,
     -1.84375,
     -2.255859375,
     -1.3759765625,
     -1.748046875,
     -2.384765625,
     -1.2412109375,
     -2.326171875,
     -2.388671875,
     -1.8984375,
     -2.412109375,
     -2.552734375,
     -0.175537109375,
     -2.6171875,
     -1.76953125,
     -1.5498046875,
     -0.70068359375,
     -1.2275390625,
     -3.126953125,
     -1.4443359375,
     -2.412109375,
     -2.337890625,
     -1.7705078125,
     -2.501953125,
     -1.767578125,
     -1.2666015625,
     -1.2392578125,
     -0.654296875,
     -0.09893798828125,
     -0.2010498046875,
     -0.95703125,
     0.6142578125,
     0.448486328125,
     -0.2802734375,
     -0.264404296875,
     -0.53662109375,
     -0.01229095458984375,
     -0.85009765625,
     -0.2169189453125,
     1.2724609375,
     1.03125,
     -0.267333984375,
     -0.2890625,
     -0.70068359375,
     0.383544921875,
     0.6748046875,
     -0.479248046875,
     0.04693603515625,
     -0.154296875,
     0.48486328125,
     0.69775390625,
     -0.33935546875,
     0.00362396240234375,
     1.916015625,
     -0.658203125,
     -0.239501953125,
     -0.484619140625,
     -0.0312347412109375,
     -0.342041015625,
     -0.84375
    ],
    "raw256_argmax": 249,
    "option_argmax": 0,
    "oracle_argmax": 0,
    "argmax_agrees": true,
    "p_full": [
     0.8107138872146606,
     0.06598784029483795,
     0.0038745864294469357,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.11942368000745773
    ],
    "p": [
     0.9206628203392029,
     0.07493710517883301,
     0.004400057718157768
    ],
    "p_oracle": [
     0.9156041741371155,
     0.07993196696043015,
     0.004463807214051485
    ],
    "abstain": 0.11942368000745773,
    "abstain_oracle": 0.13591159880161285,
    "abs_delta_abstain": 0.01648791879415512,
    "oracle_margin": 0.8356721997261047,
    "max_abs_delta_p": 0.005058646202087402,
    "mean_abs_delta_p": 0.003372419159859419,
    "max_abs_delta_raw_logits": 0.10010689496994019,
    "mean_abs_delta_raw_logits": 0.01547925331396982,
    "logits_min": -8.2265625,
    "logits_max": 1.916015625,
    "raw_logits_min": -8.2265625,
    "raw_logits_max": 1.916015625,
    "raw_logits_absmax": 8.2265625,
    "oracle_logits_min": -8.236438751220703,
    "oracle_logits_max": 1.9041804075241089,
    "oracle_raw_logits_absmax": 8.236438751220703,
    "finite": true,
    "nonconstant": true,
    "wall_seconds_contended": 0.5506762090371922,
    "seconds": 0.5506762090371922,
    "full_vocab_argmax_id": 249,
    "full_vocab_argmax_is_label": false
   },
   {
    "id": "r05-dry",
    "request_id": "r05",
    "question_id": "dry",
    "type": "noul",
    "qtype_index": 2,
    "kind": "slot",
    "tokens": 73,
    "nopts": 2,
    "temperature": 1.0067671537399292,
    "raw_logits": [
     0.06463623046875,
     0.177978515625,
     -1.271484375,
     -2.2265625,
     -2.1171875,
     -2.931640625,
     -2.623046875,
     -3.029296875,
     -4.58203125,
     -2.64453125,
     -1.9736328125,
     -3.33203125,
     -3.802734375,
     -2.4296875,
     -3.12109375,
     -3.95703125,
     -4.01171875,
     -5.3359375,
     -3.3125,
     -3.138671875,
     -3.515625,
     -2.05078125,
     -5.09375,
     -6.078125,
     -3.43359375,
     -4.7578125,
     -2.728515625,
     -4.7578125,
     -3.65625,
     -5.4453125,
     -4.01953125,
     -3.962890625,
     -5.4921875,
     -4.03515625,
     -2.64453125,
     -4.68359375,
     -5.2109375,
     -5.4609375,
     -4.578125,
     -2.587890625,
     -6.34375,
     -4.1953125,
     -4.6015625,
     -5.51953125,
     -3.25390625,
     -4.50390625,
     -3.8203125,
     -4.53515625,
     -4.59765625,
     -4.1875,
     -5.10546875,
     -4.4453125,
     -3.033203125,
     -4.75390625,
     -2.765625,
     -3.439453125,
     -3.62890625,
     -4.3984375,
     -4.07421875,
     -5.515625,
     -7.203125,
     -6.38671875,
     -7.8203125,
     -5.83203125,
     -3.2890625,
     -7.17578125,
     -7.19140625,
     -2.291015625,
     -7.484375,
     -4.23828125,
     -7.52734375,
     -4.8359375,
     -2.537109375,
     -5.3671875,
     -3.724609375,
     -5.58984375,
     -4.26953125,
     -6.27734375,
     -3.59375,
     -5.43359375,
     -4.140625,
     -3.49609375,
     -2.38671875,
     -4.0625,
     -3.2890625,
     -3.953125,
     -2.458984375,
     -3.39453125,
     -3.076171875,
     -4.16796875,
     -3.150390625,
     -3.7421875,
     -3.390625,
     -3.755859375,
     -4.3515625,
     -4.27734375,
     -4.55859375,
     -3.357421875,
     -2.640625,
     -3.859375,
     -5.1640625,
     -4.87109375,
     -3.07421875,
     -3.896484375,
     -4.0625,
     -5.48828125,
     -3.103515625,
     -4.71484375,
     -3.84375,
     -5.3203125,
     -4.140625,
     -4.53515625,
     -4.3828125,
     -5.60546875,
     -4.2265625,
     -3.17578125,
     -4.53125,
     -5.125,
     -3.87109375,
     -3.31640625,
     -3.193359375,
     -4.07421875,
     -2.77734375,
     -3.375,
     -2.359375,
     -4.83203125,
     -5.04296875,
     -2.6328125,
     -2.576171875,
     -4.71484375,
     -5.17578125,
     -2.59375,
     -5.0859375,
     -5.8671875,
     -5.18359375,
     -4.37109375,
     -3.396484375,
     -4.1796875,
     -5.46484375,
     -2.30078125,
     -3.337890625,
     -3.953125,
     -3.00390625,
     -2.955078125,
     -4.3359375,
     -4.72265625,
     -5.53515625,
     -4.96875,
     -4.80078125,
     -3.83984375,
     -4.1328125,
     -2.30859375,
     -0.80810546875,
     -2.5859375,
     -1.966796875,
     -0.94873046875,
     -1.3828125,
     -1.5595703125,
     -2.734375,
     -0.7568359375,
     -1.76171875,
     -1.654296875,
     -0.857421875,
     -1.1416015625,
     -2.533203125,
     -1.9482421875,
     -2.361328125,
     -1.66015625,
     -3.474609375,
     -1.8642578125,
     -1.736328125,
     -2.099609375,
     -1.74609375,
     -2.58984375,
     -0.71484375,
     -1.6201171875,
     -3.185546875,
     -3.01953125,
     -2.5703125,
     -2.603515625,
     -3.09765625,
     -2.158203125,
     -2.3515625,
     -1.9130859375,
     -2.07421875,
     -1.673828125,
     -1.4013671875,
     -1.390625,
     -2.005859375,
     -1.9638671875,
     -0.4873046875,
     -1.771484375,
     -2.326171875,
     -1.2099609375,
     -1.2587890625,
     -1.587890625,
     -1.8759765625,
     -1.37109375,
     -1.6435546875,
     -0.9609375,
     -1.044921875,
     -1.63671875,
     -1.2021484375,
     -1.296875,
     -1.9794921875,
     -0.86083984375,
     -2.251953125,
     -2.4609375,
     -0.129638671875,
     -1.693359375,
     -0.66748046875,
     -1.02734375,
     -0.099365234375,
     -0.8525390625,
     -2.478515625,
     -0.7978515625,
     -1.2509765625,
     -2.123046875,
     -1.3408203125,
     -1.6376953125,
     -1.4990234375,
     -1.630859375,
     -1.359375,
     -0.806640625,
     -0.0166015625,
     0.35205078125,
     -0.4189453125,
     0.9013671875,
     0.91845703125,
     -0.263671875,
     -0.304443359375,
     -0.346923828125,
     -0.0770263671875,
     -0.94091796875,
     0.2607421875,
     1.017578125,
     1.3935546875,
     -0.339111328125,
     -0.226318359375,
     -0.65087890625,
     0.6162109375,
     0.56298828125,
     -0.45654296875,
     0.1588134765625,
     0.08050537109375,
     0.43701171875,
     0.57080078125,
     -0.54443359375,
     -0.62890625,
     1.9365234375,
     -0.3623046875,
     -0.5751953125,
     -0.341796875,
     0.352294921875,
     -0.701171875,
     -7.91796875
    ],
    "raw256_argmax": 249,
    "option_argmax": 1,
    "oracle_argmax": 1,
    "argmax_agrees": true,
    "p_full": [
     0.4718044102191925,
     0.5280256867408752,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.00016992742894217372
    ],
    "p": [
     0.4718845784664154,
     0.5281153917312622
    ],
    "p_oracle": [
     0.4691595137119293,
     0.5308405160903931
    ],
    "abstain": 0.00016992742894217372,
    "abstain_oracle": 0.00016973857418633997,
    "abs_delta_abstain": 1.88854755833745e-07,
    "oracle_margin": 0.061681002378463745,
    "max_abs_delta_p": 0.0027251243591308594,
    "mean_abs_delta_p": 0.0027250945568084717,
    "max_abs_delta_raw_logits": 0.009974956512451172,
    "mean_abs_delta_raw_logits": 0.0014678033767268062,
    "logits_min": -7.91796875,
    "logits_max": 1.9365234375,
    "raw_logits_min": -7.91796875,
    "raw_logits_max": 1.9365234375,
    "raw_logits_absmax": 7.91796875,
    "oracle_logits_min": -7.923232078552246,
    "oracle_logits_max": 1.9354100227355957,
    "oracle_raw_logits_absmax": 7.923232078552246,
    "finite": true,
    "nonconstant": true,
    "wall_seconds_contended": 0.40622041607275605,
    "seconds": 0.40622041607275605,
    "full_vocab_argmax_id": 249,
    "full_vocab_argmax_is_label": false
   },
   {
    "id": "r01-route",
    "request_id": "r01",
    "question_id": "route",
    "type": "choice",
    "qtype_index": 0,
    "kind": "slot",
    "tokens": 113,
    "nopts": 3,
    "temperature": 1.058533787727356,
    "raw_logits": [
     0.93798828125,
     -4.04296875,
     -1.77734375,
     -3.1484375,
     -3.025390625,
     -3.1171875,
     -3.091796875,
     -3.0546875,
     -4.25,
     -2.951171875,
     -3.03125,
     -2.72265625,
     -4.34375,
     -3.388671875,
     -3.658203125,
     -3.583984375,
     -3.96484375,
     -4.8984375,
     -3.572265625,
     -3.408203125,
     -3.673828125,
     -2.38671875,
     -5.4921875,
     -5.2109375,
     -3.927734375,
     -4.7734375,
     -3.419921875,
     -4.7265625,
     -4.140625,
     -5.67578125,
     -4.296875,
     -4.5234375,
     -5.80859375,
     -3.939453125,
     -3.271484375,
     -4.9296875,
     -5.73046875,
     -6.21484375,
     -4.6953125,
     -3.015625,
     -5.89453125,
     -4.79296875,
     -5.14453125,
     -5.94921875,
     -3.826171875,
     -4.9609375,
     -4.34375,
     -4.94921875,
     -5.390625,
     -4.69140625,
     -5.26953125,
     -4.62109375,
     -3.86328125,
     -5.46875,
     -3.400390625,
     -3.7734375,
     -3.9453125,
     -4.4375,
     -4.578125,
     -5.76953125,
     -7.4453125,
     -7.1953125,
     -8.125,
     -5.8125,
     -3.712890625,
     -7.5,
     -8.015625,
     -2.70703125,
     -7.85546875,
     -4.3515625,
     -7.98828125,
     -5.375,
     -2.759765625,
     -5.4140625,
     -3.76171875,
     -6.3671875,
     -4.109375,
     -7.03125,
     -4.02734375,
     -6.0546875,
     -4.65625,
     -4.25390625,
     -3.181640625,
     -4.75,
     -3.55859375,
     -4.8828125,
     -2.650390625,
     -4.10546875,
     -3.326171875,
     -4.47265625,
     -4.01171875,
     -4.15234375,
     -4.30859375,
     -3.6171875,
     -4.76953125,
     -4.1015625,
     -5.0859375,
     -3.162109375,
     -2.6640625,
     -4.7421875,
     -6.1640625,
     -4.94921875,
     -3.408203125,
     -4.33984375,
     -4.77734375,
     -5.9453125,
     -3.4765625,
     -5.23046875,
     -4.12890625,
     -6.015625,
     -4.171875,
     -4.95703125,
     -5.01953125,
     -5.78125,
     -4.6953125,
     -3.5,
     -4.77734375,
     -5.984375,
     -4.2578125,
     -3.68359375,
     -3.125,
     -4.28515625,
     -2.890625,
     -3.681640625,
     -2.197265625,
     -5.09765625,
     -5.828125,
     -2.65625,
     -2.80078125,
     -5.77734375,
     -5.67578125,
     -3.236328125,
     -5.8125,
     -6.62890625,
     -5.34765625,
     -4.94140625,
     -3.755859375,
     -4.33203125,
     -6.3515625,
     -2.697265625,
     -3.625,
     -4.32421875,
     -3.037109375,
     -2.794921875,
     -4.24609375,
     -4.91796875,
     -5.87890625,
     -5.78515625,
     -5.21484375,
     -4.51171875,
     -4.5546875,
     -2.853515625,
     -1.0400390625,
     -3.15234375,
     -1.9384765625,
     -1.4453125,
     -1.5673828125,
     -1.51953125,
     -2.634765625,
     -0.86865234375,
     -1.8623046875,
     -1.7509765625,
     -1.4150390625,
     -1.34765625,
     -2.693359375,
     -1.841796875,
     -2.9765625,
     -2.271484375,
     -3.876953125,
     -2.115234375,
     -2.2109375,
     -2.541015625,
     -1.865234375,
     -2.91796875,
     -0.94873046875,
     -1.951171875,
     -3.669921875,
     -3.4140625,
     -3.232421875,
     -2.56640625,
     -3.90625,
     -2.49609375,
     -3.078125,
     -2.763671875,
     -3.390625,
     -2.501953125,
     -2.0,
     -2.05078125,
     -2.541015625,
     -2.658203125,
     -1.2587890625,
     -2.01171875,
     -2.76953125,
     -1.8408203125,
     -2.048828125,
     -2.10546875,
     -2.076171875,
     -2.02734375,
     -2.1171875,
     -1.134765625,
     -1.3955078125,
     -2.607421875,
     -1.2646484375,
     -2.27734375,
     -2.55859375,
     -1.7783203125,
     -2.203125,
     -2.5,
     0.07110595703125,
     -2.3203125,
     -1.7548828125,
     -1.232421875,
     -0.10736083984375,
     -1.1083984375,
     -3.306640625,
     -1.12890625,
     -2.017578125,
     -2.140625,
     -1.548828125,
     -2.09765625,
     -1.2158203125,
     -1.3671875,
     -1.0185546875,
     -0.8388671875,
     -0.0960693359375,
     -0.44384765625,
     -0.76806640625,
     0.890625,
     0.74658203125,
     -0.3798828125,
     -0.0428466796875,
     -0.452392578125,
     0.1131591796875,
     -0.91845703125,
     0.08892822265625,
     1.0849609375,
     0.904296875,
     -0.081298828125,
     -0.1031494140625,
     -0.72021484375,
     0.312255859375,
     0.53759765625,
     -0.7060546875,
     0.048126220703125,
     -0.137939453125,
     0.408935546875,
     0.78076171875,
     -0.75927734375,
     -0.13330078125,
     2.06640625,
     -0.72412109375,
     -0.288818359375,
     -0.50244140625,
     -0.0106964111328125,
     -0.3037109375,
     -2.74609375
    ],
    "raw256_argmax": 249,
    "option_argmax": 0,
    "oracle_argmax": 0,
    "argmax_agrees": true,
    "p_full": [
     0.8954583406448364,
     0.008099568076431751,
     0.06886537373065948,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.027576737105846405
    ],
    "p": [
     0.9208524823188782,
     0.008329262025654316,
     0.07081831246614456
    ],
    "p_oracle": [
     0.9192458987236023,
     0.008343691937625408,
     0.072410449385643
    ],
    "abstain": 0.027576737105846405,
    "abstain_oracle": 0.02907981537282467,
    "abs_delta_abstain": 0.0015030782669782639,
    "oracle_margin": 0.8468354344367981,
    "max_abs_delta_p": 0.001606583595275879,
    "mean_abs_delta_p": 0.0010710501422484715,
    "max_abs_delta_raw_logits": 0.03141129016876221,
    "mean_abs_delta_raw_logits": 0.00686514304834418,
    "logits_min": -8.125,
    "logits_max": 2.06640625,
    "raw_logits_min": -8.125,
    "raw_logits_max": 2.06640625,
    "raw_logits_absmax": 8.125,
    "oracle_logits_min": -8.13015365600586,
    "oracle_logits_max": 2.0633702278137207,
    "oracle_raw_logits_absmax": 8.13015365600586,
    "finite": true,
    "nonconstant": true,
    "wall_seconds_contended": 1.8779243340250105,
    "seconds": 1.8779243340250105,
    "full_vocab_argmax_id": 249,
    "full_vocab_argmax_is_label": false
   },
   {
    "id": "r14-fill",
    "request_id": "r14",
    "question_id": "fill",
    "type": "score",
    "qtype_index": 1,
    "kind": "slot",
    "tokens": 87,
    "nopts": 3,
    "temperature": 1.0431410074234009,
    "raw_logits": [
     2.583984375,
     1.794921875,
     -2.443359375,
     -2.859375,
     -2.48828125,
     -3.189453125,
     -3.015625,
     -3.837890625,
     -4.06640625,
     -2.71875,
     -2.689453125,
     -3.779296875,
     -4.5,
     -2.865234375,
     -3.572265625,
     -4.08984375,
     -3.826171875,
     -4.83984375,
     -2.732421875,
     -3.369140625,
     -3.189453125,
     -1.625,
     -4.9296875,
     -6.12890625,
     -3.185546875,
     -4.5078125,
     -2.1796875,
     -4.9296875,
     -4.04296875,
     -6.5,
     -3.9296875,
     -4.53125,
     -5.6484375,
     -4.6171875,
     -3.056640625,
     -4.921875,
     -5.0,
     -5.671875,
     -4.15234375,
     -2.951171875,
     -6.703125,
     -4.4453125,
     -4.94921875,
     -5.51953125,
     -3.76953125,
     -4.67578125,
     -3.8125,
     -4.42578125,
     -4.96484375,
     -4.40625,
     -5.359375,
     -4.3828125,
     -3.0,
     -4.8359375,
     -3.0234375,
     -3.44921875,
     -3.8359375,
     -4.37890625,
     -4.359375,
     -5.8125,
     -7.265625,
     -6.50390625,
     -7.87890625,
     -5.72265625,
     -3.6484375,
     -6.984375,
     -7.2734375,
     -2.201171875,
     -7.3515625,
     -4.0625,
     -7.6484375,
     -4.796875,
     -2.78125,
     -5.4296875,
     -3.66796875,
     -5.82421875,
     -4.10546875,
     -6.5703125,
     -3.611328125,
     -5.43359375,
     -4.0859375,
     -3.693359375,
     -2.53515625,
     -4.24609375,
     -3.259765625,
     -3.8984375,
     -2.369140625,
     -3.68359375,
     -2.921875,
     -4.53125,
     -3.296875,
     -3.94140625,
     -3.734375,
     -3.6640625,
     -4.0625,
     -4.0546875,
     -4.7890625,
     -3.544921875,
     -2.998046875,
     -4.0,
     -5.4375,
     -4.75,
     -3.169921875,
     -3.900390625,
     -4.19921875,
     -5.7890625,
     -3.791015625,
     -5.17578125,
     -3.935546875,
     -5.58203125,
     -4.265625,
     -4.6953125,
     -4.76953125,
     -6.1171875,
     -4.734375,
     -3.775390625,
     -4.62109375,
     -5.48046875,
     -3.88671875,
     -3.4765625,
     -3.353515625,
     -4.0390625,
     -3.1328125,
     -3.53515625,
     -2.4140625,
     -5.19921875,
     -5.41796875,
     -2.6796875,
     -2.640625,
     -5.1171875,
     -5.40234375,
     -3.03125,
     -5.27734375,
     -5.84765625,
     -4.95703125,
     -4.15625,
     -3.529296875,
     -4.8828125,
     -5.6328125,
     -2.8515625,
     -3.53515625,
     -4.09375,
     -3.15625,
     -2.68359375,
     -4.80078125,
     -5.12109375,
     -5.77734375,
     -4.9921875,
     -4.7421875,
     -4.21875,
     -4.33203125,
     -2.7265625,
     -1.0126953125,
     -3.080078125,
     -2.599609375,
     -1.361328125,
     -1.7841796875,
     -1.912109375,
     -2.982421875,
     -0.72412109375,
     -1.830078125,
     -2.0,
     -1.544921875,
     -1.1259765625,
     -2.576171875,
     -1.939453125,
     -2.666015625,
     -2.125,
     -3.458984375,
     -2.033203125,
     -2.10546875,
     -2.486328125,
     -2.166015625,
     -2.9296875,
     -0.7900390625,
     -2.109375,
     -3.576171875,
     -3.375,
     -2.90625,
     -2.583984375,
     -3.224609375,
     -2.6484375,
     -2.486328125,
     -2.15234375,
     -2.197265625,
     -1.728515625,
     -1.37890625,
     -1.4892578125,
     -2.3671875,
     -2.26171875,
     -0.56298828125,
     -1.640625,
     -2.607421875,
     -1.3642578125,
     -1.4501953125,
     -1.5283203125,
     -2.06640625,
     -1.595703125,
     -1.8466796875,
     -1.009765625,
     -0.96484375,
     -1.7490234375,
     -1.3896484375,
     -1.5595703125,
     -2.26171875,
     -0.94482421875,
     -2.46875,
     -2.640625,
     -0.37646484375,
     -2.248046875,
     -1.154296875,
     -1.24609375,
     -0.206787109375,
     -0.99560546875,
     -2.4765625,
     -1.0693359375,
     -1.62109375,
     -2.16015625,
     -1.2255859375,
     -1.955078125,
     -1.3544921875,
     -1.7939453125,
     -1.0849609375,
     -0.91162109375,
     -0.2476806640625,
     0.09613037109375,
     -0.53173828125,
     1.09375,
     0.8486328125,
     -0.29345703125,
     -0.477783203125,
     -0.744140625,
     0.0833740234375,
     -0.89599609375,
     0.11138916015625,
     0.908203125,
     1.3984375,
     -0.313232421875,
     0.23828125,
     -0.8427734375,
     0.361083984375,
     0.58251953125,
     -0.5439453125,
     -0.1513671875,
     -0.09283447265625,
     0.467041015625,
     0.4833984375,
     -0.242431640625,
     -0.8935546875,
     2.02734375,
     -0.3603515625,
     -0.449462890625,
     -0.2080078125,
     0.328369140625,
     -0.57421875,
     -5.29296875
    ],
    "raw256_argmax": 0,
    "option_argmax": 0,
    "oracle_argmax": 0,
    "argmax_agrees": true,
    "p_full": [
     0.6766192317008972,
     0.3175639510154724,
     0.005461263936012983,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.00035555375507101417
    ],
    "p": [
     0.6768598556518555,
     0.3176769018173218,
     0.005463206209242344
    ],
    "p_oracle": [
     0.675521969795227,
     0.3190520703792572,
     0.005425968673080206
    ],
    "abstain": 0.00035555375507101417,
    "abstain_oracle": 0.000354081392288208,
    "abs_delta_abstain": 1.472362782806158e-06,
    "oracle_margin": 0.35646989941596985,
    "max_abs_delta_p": 0.0013751685619354248,
    "mean_abs_delta_p": 0.0009167639849086603,
    "max_abs_delta_raw_logits": 0.008677959442138672,
    "mean_abs_delta_raw_logits": 0.0017450845334678888,
    "logits_min": -7.87890625,
    "logits_max": 2.583984375,
    "raw_logits_min": -7.87890625,
    "raw_logits_max": 2.583984375,
    "raw_logits_absmax": 7.87890625,
    "oracle_logits_min": -7.882862567901611,
    "oracle_logits_max": 2.586092710494995,
    "oracle_raw_logits_absmax": 7.882862567901611,
    "finite": true,
    "nonconstant": true,
    "wall_seconds_contended": 0.4859221250517294,
    "seconds": 0.4859221250517294,
    "full_vocab_argmax_id": 0,
    "full_vocab_argmax_is_label": true
   },
   {
    "id": "r01-condition",
    "request_id": "r01",
    "question_id": "condition",
    "type": "choice",
    "qtype_index": 0,
    "kind": "slot",
    "tokens": 87,
    "nopts": 4,
    "temperature": 1.058533787727356,
    "raw_logits": [
     -3.24609375,
     3.130859375,
     -2.6171875,
     -0.74267578125,
     -2.30078125,
     -3.001953125,
     -2.83984375,
     -3.623046875,
     -4.10546875,
     -3.08984375,
     -3.048828125,
     -3.9765625,
     -5.40234375,
     -3.068359375,
     -4.16796875,
     -4.73046875,
     -4.15234375,
     -4.44921875,
     -2.947265625,
     -2.98046875,
     -3.341796875,
     -1.6533203125,
     -5.2578125,
     -6.63671875,
     -4.109375,
     -5.39453125,
     -2.58984375,
     -5.8203125,
     -4.40234375,
     -6.65625,
     -4.67578125,
     -4.875,
     -6.25,
     -4.609375,
     -2.935546875,
     -5.47265625,
     -5.1640625,
     -5.6171875,
     -4.01171875,
     -2.943359375,
     -7.26953125,
     -4.25390625,
     -4.93359375,
     -5.90234375,
     -4.42578125,
     -5.07421875,
     -4.91796875,
     -4.80859375,
     -5.3046875,
     -4.60546875,
     -5.69140625,
     -4.8515625,
     -2.439453125,
     -5.05859375,
     -2.44140625,
     -3.08203125,
     -3.759765625,
     -4.49609375,
     -4.72265625,
     -5.73046875,
     -7.65234375,
     -6.4453125,
     -7.5859375,
     -6.55859375,
     -3.751953125,
     -7.55078125,
     -7.59765625,
     -2.166015625,
     -7.74609375,
     -4.26171875,
     -8.171875,
     -5.37109375,
     -2.5546875,
     -6.31640625,
     -3.861328125,
     -6.36328125,
     -4.38671875,
     -6.67578125,
     -4.13671875,
     -5.5703125,
     -4.1875,
     -4.046875,
     -2.46875,
     -3.966796875,
     -3.40625,
     -4.0234375,
     -2.720703125,
     -3.62890625,
     -3.056640625,
     -4.87109375,
     -3.578125,
     -4.1328125,
     -4.20703125,
     -4.34765625,
     -4.71875,
     -4.94140625,
     -5.203125,
     -4.109375,
     -4.14453125,
     -3.912109375,
     -6.00390625,
     -4.8828125,
     -3.48046875,
     -4.359375,
     -4.37890625,
     -6.21484375,
     -4.34375,
     -5.66015625,
     -4.29296875,
     -5.32421875,
     -4.45703125,
     -4.79296875,
     -5.1875,
     -6.59375,
     -4.93359375,
     -4.30078125,
     -4.6171875,
     -6.109375,
     -4.60546875,
     -3.333984375,
     -3.1640625,
     -4.12109375,
     -3.314453125,
     -3.841796875,
     -2.52734375,
     -5.953125,
     -5.78515625,
     -3.37109375,
     -3.267578125,
     -5.515625,
     -5.5234375,
     -3.7890625,
     -5.7578125,
     -6.08984375,
     -5.07421875,
     -4.74609375,
     -4.05078125,
     -5.03125,
     -5.8828125,
     -2.94921875,
     -3.73828125,
     -4.1953125,
     -3.78515625,
     -3.5234375,
     -4.796875,
     -5.27734375,
     -6.2890625,
     -5.26953125,
     -4.88671875,
     -4.40234375,
     -4.78125,
     -3.1484375,
     -1.3310546875,
     -3.67578125,
     -2.64453125,
     -1.798828125,
     -2.021484375,
     -2.154296875,
     -3.423828125,
     -0.791015625,
     -2.40234375,
     -2.259765625,
     -1.751953125,
     -1.35546875,
     -2.78125,
     -2.400390625,
     -2.865234375,
     -1.7333984375,
     -3.369140625,
     -2.197265625,
     -2.0859375,
     -2.59375,
     -2.51171875,
     -3.12109375,
     -1.162109375,
     -2.013671875,
     -3.25,
     -3.41796875,
     -3.0859375,
     -3.646484375,
     -3.546875,
     -3.271484375,
     -2.693359375,
     -2.021484375,
     -2.638671875,
     -1.5185546875,
     -1.40625,
     -1.787109375,
     -1.927734375,
     -1.896484375,
     -1.212890625,
     -1.689453125,
     -3.013671875,
     -1.5771484375,
     -1.3349609375,
     -2.138671875,
     -1.9267578125,
     -1.8916015625,
     -1.73046875,
     -1.3154296875,
     -1.3759765625,
     -1.951171875,
     -1.908203125,
     -1.7978515625,
     -2.353515625,
     -1.1259765625,
     -2.6640625,
     -2.734375,
     -0.384033203125,
     -2.302734375,
     -0.7998046875,
     -1.333984375,
     -0.82421875,
     -1.494140625,
     -2.447265625,
     -1.4921875,
     -1.8623046875,
     -2.443359375,
     -1.86328125,
     -2.0390625,
     -1.392578125,
     -1.736328125,
     -1.5703125,
     -1.416015625,
     -0.1927490234375,
     0.2607421875,
     -0.342529296875,
     0.73779296875,
     0.94677734375,
     -0.10662841796875,
     -1.0830078125,
     -0.492431640625,
     -0.1053466796875,
     -0.91259765625,
     0.85302734375,
     1.13671875,
     1.552734375,
     -0.6640625,
     0.057373046875,
     -0.76806640625,
     0.32763671875,
     0.6875,
     -0.775390625,
     -0.0853271484375,
     0.144775390625,
     0.54736328125,
     0.2430419921875,
     -0.324951171875,
     -0.65185546875,
     1.4921875,
     0.042083740234375,
     -0.744140625,
     -0.421630859375,
     0.5478515625,
     -0.5576171875,
     -4.60546875
    ],
    "raw256_argmax": 1,
    "option_argmax": 1,
    "oracle_argmax": 1,
    "argmax_agrees": true,
    "p_full": [
     0.0023413978051394224,
     0.967847466468811,
     0.004241333808749914,
     0.024921584874391556,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0,
     0.0006482636672444642
    ],
    "p": [
     0.0023429165594279766,
     0.9684752225875854,
     0.004244084935635328,
     0.024937748908996582
    ],
    "p_oracle": [
     0.0023138143587857485,
     0.9696792364120483,
     0.004179855342954397,
     0.023827005177736282
    ],
    "abstain": 0.0006482636672444642,
    "abstain_oracle": 0.000638672907371074,
    "abs_delta_abstain": 9.590759873390198e-06,
    "oracle_margin": 0.9458522200584412,
    "max_abs_delta_p": 0.0012040138244628906,
    "mean_abs_delta_p": 0.0006020223372615874,
    "max_abs_delta_raw_logits": 0.03208816051483154,
    "mean_abs_delta_raw_logits": 0.004462839162442833,
    "logits_min": -8.171875,
    "logits_max": 3.130859375,
    "raw_logits_min": -8.171875,
    "raw_logits_max": 3.130859375,
    "raw_logits_absmax": 8.171875,
    "oracle_logits_min": -8.181787490844727,
    "oracle_logits_max": 3.1483166217803955,
    "oracle_raw_logits_absmax": 8.181787490844727,
    "finite": true,
    "nonconstant": true,
    "wall_seconds_contended": 0.4949088330613449,
    "seconds": 0.4949088330613449,
    "full_vocab_argmax_id": 1,
    "full_vocab_argmax_is_label": true
   }
  ]
 },
 "reset_check": {
  "row": "r01-route",
  "identical": true,
  "max_abs_diff": 0.0,
  "final_session": "fp16-04-global_reset-a1",
  "final_raw_logits": [
   0.93798828125,
   -4.04296875,
   -1.77734375,
   -3.1484375,
   -3.025390625,
   -3.1171875,
   -3.091796875,
   -3.0546875,
   -4.25,
   -2.951171875,
   -3.03125,
   -2.72265625,
   -4.34375,
   -3.388671875,
   -3.658203125,
   -3.583984375,
   -3.96484375,
   -4.8984375,
   -3.572265625,
   -3.408203125,
   -3.673828125,
   -2.38671875,
   -5.4921875,
   -5.2109375,
   -3.927734375,
   -4.7734375,
   -3.419921875,
   -4.7265625,
   -4.140625,
   -5.67578125,
   -4.296875,
   -4.5234375,
   -5.80859375,
   -3.939453125,
   -3.271484375,
   -4.9296875,
   -5.73046875,
   -6.21484375,
   -4.6953125,
   -3.015625,
   -5.89453125,
   -4.79296875,
   -5.14453125,
   -5.94921875,
   -3.826171875,
   -4.9609375,
   -4.34375,
   -4.94921875,
   -5.390625,
   -4.69140625,
   -5.26953125,
   -4.62109375,
   -3.86328125,
   -5.46875,
   -3.400390625,
   -3.7734375,
   -3.9453125,
   -4.4375,
   -4.578125,
   -5.76953125,
   -7.4453125,
   -7.1953125,
   -8.125,
   -5.8125,
   -3.712890625,
   -7.5,
   -8.015625,
   -2.70703125,
   -7.85546875,
   -4.3515625,
   -7.98828125,
   -5.375,
   -2.759765625,
   -5.4140625,
   -3.76171875,
   -6.3671875,
   -4.109375,
   -7.03125,
   -4.02734375,
   -6.0546875,
   -4.65625,
   -4.25390625,
   -3.181640625,
   -4.75,
   -3.55859375,
   -4.8828125,
   -2.650390625,
   -4.10546875,
   -3.326171875,
   -4.47265625,
   -4.01171875,
   -4.15234375,
   -4.30859375,
   -3.6171875,
   -4.76953125,
   -4.1015625,
   -5.0859375,
   -3.162109375,
   -2.6640625,
   -4.7421875,
   -6.1640625,
   -4.94921875,
   -3.408203125,
   -4.33984375,
   -4.77734375,
   -5.9453125,
   -3.4765625,
   -5.23046875,
   -4.12890625,
   -6.015625,
   -4.171875,
   -4.95703125,
   -5.01953125,
   -5.78125,
   -4.6953125,
   -3.5,
   -4.77734375,
   -5.984375,
   -4.2578125,
   -3.68359375,
   -3.125,
   -4.28515625,
   -2.890625,
   -3.681640625,
   -2.197265625,
   -5.09765625,
   -5.828125,
   -2.65625,
   -2.80078125,
   -5.77734375,
   -5.67578125,
   -3.236328125,
   -5.8125,
   -6.62890625,
   -5.34765625,
   -4.94140625,
   -3.755859375,
   -4.33203125,
   -6.3515625,
   -2.697265625,
   -3.625,
   -4.32421875,
   -3.037109375,
   -2.794921875,
   -4.24609375,
   -4.91796875,
   -5.87890625,
   -5.78515625,
   -5.21484375,
   -4.51171875,
   -4.5546875,
   -2.853515625,
   -1.0400390625,
   -3.15234375,
   -1.9384765625,
   -1.4453125,
   -1.5673828125,
   -1.51953125,
   -2.634765625,
   -0.86865234375,
   -1.8623046875,
   -1.7509765625,
   -1.4150390625,
   -1.34765625,
   -2.693359375,
   -1.841796875,
   -2.9765625,
   -2.271484375,
   -3.876953125,
   -2.115234375,
   -2.2109375,
   -2.541015625,
   -1.865234375,
   -2.91796875,
   -0.94873046875,
   -1.951171875,
   -3.669921875,
   -3.4140625,
   -3.232421875,
   -2.56640625,
   -3.90625,
   -2.49609375,
   -3.078125,
   -2.763671875,
   -3.390625,
   -2.501953125,
   -2.0,
   -2.05078125,
   -2.541015625,
   -2.658203125,
   -1.2587890625,
   -2.01171875,
   -2.76953125,
   -1.8408203125,
   -2.048828125,
   -2.10546875,
   -2.076171875,
   -2.02734375,
   -2.1171875,
   -1.134765625,
   -1.3955078125,
   -2.607421875,
   -1.2646484375,
   -2.27734375,
   -2.55859375,
   -1.7783203125,
   -2.203125,
   -2.5,
   0.07110595703125,
   -2.3203125,
   -1.7548828125,
   -1.232421875,
   -0.10736083984375,
   -1.1083984375,
   -3.306640625,
   -1.12890625,
   -2.017578125,
   -2.140625,
   -1.548828125,
   -2.09765625,
   -1.2158203125,
   -1.3671875,
   -1.0185546875,
   -0.8388671875,
   -0.0960693359375,
   -0.44384765625,
   -0.76806640625,
   0.890625,
   0.74658203125,
   -0.3798828125,
   -0.0428466796875,
   -0.452392578125,
   0.1131591796875,
   -0.91845703125,
   0.08892822265625,
   1.0849609375,
   0.904296875,
   -0.081298828125,
   -0.1031494140625,
   -0.72021484375,
   0.312255859375,
   0.53759765625,
   -0.7060546875,
   0.048126220703125,
   -0.137939453125,
   0.408935546875,
   0.78076171875,
   -0.75927734375,
   -0.13330078125,
   2.06640625,
   -0.72412109375,
   -0.288818359375,
   -0.50244140625,
   -0.0106964111328125,
   -0.3037109375,
   -2.74609375
  ],
  "cross_process": true,
  "local_resets_in_every_session": true
 },
 "wall_seconds_contended": 65.10267804202158,
 "completed_at": "2026-09-22T19:27:32+00:00",
 "generated_at": "2026-09-22T19:27:32+00:00"
}