{
 "schema": "coreai-scalar-readout-gate/1",
 "mode": "fp16",
 "bundle": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/exports/system_one_qwen3_5_4b_scorer_decode_fp16",
 "fixtures": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/oracle-fixtures.json",
 "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
 "max_ctx": 4096,
 "runtime": "coreai Python runtime, AOT h16c GPU asset, SpecializationOptions.default()",
 "source": null,
 "temperature": 1.75,
 "max_len": 384,
 "layout": "State/Question/Option, last token",
 "logits_width": 1,
 "license": "cc-by-nc-4.0",
 "environment": {
  "round": 3,
  "start_epoch": 1790120901.0,
  "deadline_epoch": 1790129901.0,
  "start_utc": "2026-09-22T23:48:21+00:00",
  "deadline_utc": "2026-09-23T02:18:21+00:00",
  "limit_seconds": 9000,
  "fence": "OPEN",
  "device": "Apple M4 Max, 128 GiB RAM, macOS 27.0 (26A428)",
  "license": "cc-by-nc-4.0",
  "timing": "A: quiet-window GPU throughput; B: correctness only",
  "result": "RUNNING"
 },
 "started_at": "2026-09-23T00:42:05+00:00",
 "timing": "contended; correctness only, no performance claim",
 "rows_per_worker": 14,
 "max_row_executions_per_worker": 15,
 "rows": [
  {
   "id": "request-01-q01-option-00",
   "request_id": "request-01",
   "question_id": "request-01-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": 4.3125,
   "scalar_oracle": 4.314200401306152,
   "abs_delta_scalar": 0.0017004013061523438,
   "scalar_dtype": "<f2",
   "scalar_bits": "5044",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 34.572489249985665
  },
  {
   "id": "request-01-q01-option-01",
   "request_id": "request-01",
   "question_id": "request-01-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -8.171875,
   "scalar_oracle": -8.20827865600586,
   "abs_delta_scalar": 0.036403656005859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "16c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.865309958113357
  },
  {
   "id": "request-02-q01-option-00",
   "request_id": "request-02",
   "question_id": "request-02-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": 6.23828125,
   "scalar_oracle": 6.207533836364746,
   "abs_delta_scalar": 0.030747413635253906,
   "scalar_dtype": "<f2",
   "scalar_bits": "3d46",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.940669041941874
  },
  {
   "id": "request-02-q01-option-01",
   "request_id": "request-02",
   "question_id": "request-02-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -9.421875,
   "scalar_oracle": -9.434723854064941,
   "abs_delta_scalar": 0.012848854064941406,
   "scalar_dtype": "<f2",
   "scalar_bits": "b6c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.897903250064701
  },
  {
   "id": "request-03-q01-option-00",
   "request_id": "request-03",
   "question_id": "request-03-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": 6.625,
   "scalar_oracle": 6.595953941345215,
   "abs_delta_scalar": 0.029046058654785156,
   "scalar_dtype": "<f2",
   "scalar_bits": "a046",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.880011041997932
  },
  {
   "id": "request-03-q01-option-01",
   "request_id": "request-03",
   "question_id": "request-03-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -8.2109375,
   "scalar_oracle": -8.22585678100586,
   "abs_delta_scalar": 0.014919281005859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "1bc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.846642832970247
  },
  {
   "id": "request-04-q01-option-00",
   "request_id": "request-04",
   "question_id": "request-04-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": 5.984375,
   "scalar_oracle": 5.953964710235596,
   "abs_delta_scalar": 0.030410289764404297,
   "scalar_dtype": "<f2",
   "scalar_bits": "fc45",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.877799541922286
  },
  {
   "id": "request-04-q01-option-01",
   "request_id": "request-04",
   "question_id": "request-04-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -10.546875,
   "scalar_oracle": -10.532769203186035,
   "abs_delta_scalar": 0.014105796813964844,
   "scalar_dtype": "<f2",
   "scalar_bits": "46c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.863865416031331
  },
  {
   "id": "request-05-q01-option-00",
   "request_id": "request-05",
   "question_id": "request-05-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": 7.65234375,
   "scalar_oracle": 7.658699989318848,
   "abs_delta_scalar": 0.006356239318847656,
   "scalar_dtype": "<f2",
   "scalar_bits": "a747",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.896409041015431
  },
  {
   "id": "request-05-q01-option-01",
   "request_id": "request-05",
   "question_id": "request-05-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -8.84375,
   "scalar_oracle": -8.862298965454102,
   "abs_delta_scalar": 0.018548965454101562,
   "scalar_dtype": "<f2",
   "scalar_bits": "6cc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.895321082905866
  },
  {
   "id": "request-06-q01-option-00",
   "request_id": "request-06",
   "question_id": "request-06-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -0.779296875,
   "scalar_oracle": -0.843937337398529,
   "abs_delta_scalar": 0.06464046239852905,
   "scalar_dtype": "<f2",
   "scalar_bits": "3cba",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.884147500037216
  },
  {
   "id": "request-06-q01-option-01",
   "request_id": "request-06",
   "question_id": "request-06-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -8.265625,
   "scalar_oracle": -8.322973251342773,
   "abs_delta_scalar": 0.05734825134277344,
   "scalar_dtype": "<f2",
   "scalar_bits": "22c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.856768500059843
  },
  {
   "id": "request-07-q01-option-00",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 4.14453125,
   "scalar_oracle": 4.116079330444336,
   "abs_delta_scalar": 0.028451919555664062,
   "scalar_dtype": "<f2",
   "scalar_bits": "2544",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5448680409463122
  },
  {
   "id": "request-07-q01-option-01",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.671875,
   "scalar_oracle": -9.69235897064209,
   "abs_delta_scalar": 0.020483970642089844,
   "scalar_dtype": "<f2",
   "scalar_bits": "d6c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.566119333030656
  },
  {
   "id": "request-07-q01-option-02",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.296875,
   "scalar_oracle": -9.30652904510498,
   "abs_delta_scalar": 0.009654045104980469,
   "scalar_dtype": "<f2",
   "scalar_bits": "a6c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 13.08329550002236
  },
  {
   "id": "request-07-q01-option-03",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -5.69140625,
   "scalar_oracle": -5.676880836486816,
   "abs_delta_scalar": 0.014525413513183594,
   "scalar_dtype": "<f2",
   "scalar_bits": "b1c5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1705530839972198
  },
  {
   "id": "request-07-q01-option-04",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.5859375,
   "scalar_oracle": -10.599185943603516,
   "abs_delta_scalar": 0.013248443603515625,
   "scalar_dtype": "<f2",
   "scalar_bits": "4bc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0945161660201848
  },
  {
   "id": "request-07-q01-option-05",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.9609375,
   "scalar_oracle": -8.972684860229492,
   "abs_delta_scalar": 0.011747360229492188,
   "scalar_dtype": "<f2",
   "scalar_bits": "7bc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1115673329913989
  },
  {
   "id": "request-07-q01-option-06",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.8046875,
   "scalar_oracle": -9.825861930847168,
   "abs_delta_scalar": 0.02117443084716797,
   "scalar_dtype": "<f2",
   "scalar_bits": "e7c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1471456669969484
  },
  {
   "id": "request-07-q01-option-07",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.8125,
   "scalar_oracle": -6.823195457458496,
   "abs_delta_scalar": 0.010695457458496094,
   "scalar_dtype": "<f2",
   "scalar_bits": "d0c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1056256669107825
  },
  {
   "id": "request-07-q01-option-08",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.78125,
   "scalar_oracle": -6.788428783416748,
   "abs_delta_scalar": 0.007178783416748047,
   "scalar_dtype": "<f2",
   "scalar_bits": "c8c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0947380419820547
  },
  {
   "id": "request-07-q02-option-00",
   "request_id": "request-07",
   "question_id": "request-07-q02",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": 3.23046875,
   "scalar_oracle": 3.2154455184936523,
   "abs_delta_scalar": 0.015023231506347656,
   "scalar_dtype": "<f2",
   "scalar_bits": "7642",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1688409579219297
  },
  {
   "id": "request-07-q02-option-01",
   "request_id": "request-07",
   "question_id": "request-07-q02",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": -1.458984375,
   "scalar_oracle": -1.4813950061798096,
   "abs_delta_scalar": 0.02241063117980957,
   "scalar_dtype": "<f2",
   "scalar_bits": "d6bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1787554579786956
  },
  {
   "id": "request-07-q03-option-00",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": -0.18896484375,
   "scalar_oracle": -0.1996888518333435,
   "abs_delta_scalar": 0.010724008083343506,
   "scalar_dtype": "<f2",
   "scalar_bits": "0cb2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.5969099170761183
  },
  {
   "id": "request-07-q03-option-01",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 1.06640625,
   "scalar_oracle": 1.066279649734497,
   "abs_delta_scalar": 0.0001266002655029297,
   "scalar_dtype": "<f2",
   "scalar_bits": "443c",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.5943485419265926
  },
  {
   "id": "request-07-q03-option-02",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.98291015625,
   "scalar_oracle": 0.9806243181228638,
   "abs_delta_scalar": 0.0022858381271362305,
   "scalar_dtype": "<f2",
   "scalar_bits": "dd3b",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.5938222500262782
  },
  {
   "id": "request-07-q03-option-03",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.75390625,
   "scalar_oracle": 0.7489750385284424,
   "abs_delta_scalar": 0.004931211471557617,
   "scalar_dtype": "<f2",
   "scalar_bits": "083a",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.6083791250130162
  },
  {
   "id": "request-07-q03-option-04",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.443115234375,
   "scalar_oracle": 0.4373612403869629,
   "abs_delta_scalar": 0.005753993988037109,
   "scalar_dtype": "<f2",
   "scalar_bits": "1737",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.6183306670282036
  },
  {
   "id": "request-07-q03-option-05",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.7021484375,
   "scalar_oracle": 0.6929409503936768,
   "abs_delta_scalar": 0.009207487106323242,
   "scalar_dtype": "<f2",
   "scalar_bits": "9e39",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 3.2898249169811606
  },
  {
   "id": "request-07-q03-option-06",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.765625,
   "scalar_oracle": 0.7530665397644043,
   "abs_delta_scalar": 0.012558460235595703,
   "scalar_dtype": "<f2",
   "scalar_bits": "203a",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.628412124933675
  },
  {
   "id": "request-07-q03-option-07",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.6796875,
   "scalar_oracle": 0.6613963842391968,
   "abs_delta_scalar": 0.018291115760803223,
   "scalar_dtype": "<f2",
   "scalar_bits": "7039",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.6093820419628173
  },
  {
   "id": "request-07-q03-option-08",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": -0.2161865234375,
   "scalar_oracle": -0.23028063774108887,
   "abs_delta_scalar": 0.014094114303588867,
   "scalar_dtype": "<f2",
   "scalar_bits": "ebb2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.5926639579702169
  },
  {
   "id": "request-07-q03-option-09",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 39,
   "slot": 38,
   "scalar": -1.689453125,
   "scalar_oracle": -1.719403862953186,
   "abs_delta_scalar": 0.029950737953186035,
   "scalar_dtype": "<f2",
   "scalar_bits": "c2be",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.6233262909809127
  },
  {
   "id": "request-08-q01-option-00",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 0.93359375,
   "scalar_oracle": 0.9249671101570129,
   "abs_delta_scalar": 0.00862663984298706,
   "scalar_dtype": "<f2",
   "scalar_bits": "783b",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0747212499845773
  },
  {
   "id": "request-08-q01-option-01",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.0546875,
   "scalar_oracle": -9.055648803710938,
   "abs_delta_scalar": 0.0009613037109375,
   "scalar_dtype": "<f2",
   "scalar_bits": "87c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0751355000538751
  },
  {
   "id": "request-08-q01-option-02",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.87890625,
   "scalar_oracle": -7.874567985534668,
   "abs_delta_scalar": 0.004338264465332031,
   "scalar_dtype": "<f2",
   "scalar_bits": "e1c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0934892080258578
  },
  {
   "id": "request-08-q01-option-03",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -11.46875,
   "scalar_oracle": -11.47532844543457,
   "abs_delta_scalar": 0.0065784454345703125,
   "scalar_dtype": "<f2",
   "scalar_bits": "bcc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1265673330053687
  },
  {
   "id": "request-08-q01-option-04",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.28515625,
   "scalar_oracle": -7.288543701171875,
   "abs_delta_scalar": 0.003387451171875,
   "scalar_dtype": "<f2",
   "scalar_bits": "49c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0893746248912066
  },
  {
   "id": "request-08-q01-option-05",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.6640625,
   "scalar_oracle": -9.6605863571167,
   "abs_delta_scalar": 0.0034761428833007812,
   "scalar_dtype": "<f2",
   "scalar_bits": "d5c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1145051249768585
  },
  {
   "id": "request-08-q01-option-06",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.515625,
   "scalar_oracle": -9.511998176574707,
   "abs_delta_scalar": 0.0036268234252929688,
   "scalar_dtype": "<f2",
   "scalar_bits": "c2c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.094125457922928
  },
  {
   "id": "request-08-q01-option-07",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.34375,
   "scalar_oracle": -10.34797191619873,
   "abs_delta_scalar": 0.004221916198730469,
   "scalar_dtype": "<f2",
   "scalar_bits": "2cc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1140728750033304
  },
  {
   "id": "request-08-q01-option-08",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.578125,
   "scalar_oracle": -9.580318450927734,
   "abs_delta_scalar": 0.002193450927734375,
   "scalar_dtype": "<f2",
   "scalar_bits": "cac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1046014169696718
  },
  {
   "id": "request-08-q01-option-09",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -10.6953125,
   "scalar_oracle": -10.699100494384766,
   "abs_delta_scalar": 0.003787994384765625,
   "scalar_dtype": "<f2",
   "scalar_bits": "59c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.7968047079630196
  },
  {
   "id": "request-08-q02-option-00",
   "request_id": "request-08",
   "question_id": "request-08-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": 3.578125,
   "scalar_oracle": 3.5709025859832764,
   "abs_delta_scalar": 0.007222414016723633,
   "scalar_dtype": "<f2",
   "scalar_bits": "2843",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.084672832977958
  },
  {
   "id": "request-08-q02-option-01",
   "request_id": "request-08",
   "question_id": "request-08-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": -1.326171875,
   "scalar_oracle": -1.3289239406585693,
   "abs_delta_scalar": 0.002752065658569336,
   "scalar_dtype": "<f2",
   "scalar_bits": "4ebd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.105544958030805
  },
  {
   "id": "request-08-q03-option-00",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -3.30859375,
   "scalar_oracle": -3.3124825954437256,
   "abs_delta_scalar": 0.003888845443725586,
   "scalar_dtype": "<f2",
   "scalar_bits": "9ec2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.7342958750668913
  },
  {
   "id": "request-08-q03-option-01",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.8974609375,
   "scalar_oracle": -1.9047918319702148,
   "abs_delta_scalar": 0.007330894470214844,
   "scalar_dtype": "<f2",
   "scalar_bits": "97bf",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.7297191249672323
  },
  {
   "id": "request-08-q03-option-02",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.4248046875,
   "scalar_oracle": -1.4303022623062134,
   "abs_delta_scalar": 0.005497574806213379,
   "scalar_dtype": "<f2",
   "scalar_bits": "b3bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.7367160000139847
  },
  {
   "id": "request-08-q03-option-03",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.3857421875,
   "scalar_oracle": -1.3922051191329956,
   "abs_delta_scalar": 0.0064629316329956055,
   "scalar_dtype": "<f2",
   "scalar_bits": "8bbd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.7635809580096975
  },
  {
   "id": "request-08-q03-option-04",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.482421875,
   "scalar_oracle": -1.48871910572052,
   "abs_delta_scalar": 0.0062972307205200195,
   "scalar_dtype": "<f2",
   "scalar_bits": "eebd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.7555200420320034
  },
  {
   "id": "request-08-q03-option-05",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.203125,
   "scalar_oracle": -1.2124919891357422,
   "abs_delta_scalar": 0.009366989135742188,
   "scalar_dtype": "<f2",
   "scalar_bits": "d0bc",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.7021945420419797
  },
  {
   "id": "request-08-q03-option-06",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.2607421875,
   "scalar_oracle": -1.2709437608718872,
   "abs_delta_scalar": 0.010201573371887207,
   "scalar_dtype": "<f2",
   "scalar_bits": "0bbd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.751678542001173
  },
  {
   "id": "request-08-q03-option-07",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.3466796875,
   "scalar_oracle": -1.354970097541809,
   "abs_delta_scalar": 0.008290410041809082,
   "scalar_dtype": "<f2",
   "scalar_bits": "63bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.7504249999765307
  },
  {
   "id": "request-08-q03-option-08",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.4814453125,
   "scalar_oracle": -1.4879703521728516,
   "abs_delta_scalar": 0.0065250396728515625,
   "scalar_dtype": "<f2",
   "scalar_bits": "edbd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.7690445830812678
  },
  {
   "id": "request-08-q03-option-09",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 43,
   "slot": 42,
   "scalar": -2.38671875,
   "scalar_oracle": -2.396772861480713,
   "abs_delta_scalar": 0.01005411148071289,
   "scalar_dtype": "<f2",
   "scalar_bits": "c6c0",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.8346670829923823
  },
  {
   "id": "request-09-q01-option-00",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 0.79541015625,
   "scalar_oracle": 0.7694981694221497,
   "abs_delta_scalar": 0.025911986827850342,
   "scalar_dtype": "<f2",
   "scalar_bits": "5d3a",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.099099582992494
  },
  {
   "id": "request-09-q01-option-01",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.54296875,
   "scalar_oracle": -7.545819282531738,
   "abs_delta_scalar": 0.0028505325317382812,
   "scalar_dtype": "<f2",
   "scalar_bits": "8bc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.7608182911062613
  },
  {
   "id": "request-09-q01-option-02",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.2890625,
   "scalar_oracle": -8.288448333740234,
   "abs_delta_scalar": 0.000614166259765625,
   "scalar_dtype": "<f2",
   "scalar_bits": "25c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1091246670112014
  },
  {
   "id": "request-09-q01-option-03",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.6171875,
   "scalar_oracle": -8.621004104614258,
   "abs_delta_scalar": 0.0038166046142578125,
   "scalar_dtype": "<f2",
   "scalar_bits": "4fc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0839301249943674
  },
  {
   "id": "request-09-q01-option-04",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.46875,
   "scalar_oracle": -8.477935791015625,
   "abs_delta_scalar": 0.009185791015625,
   "scalar_dtype": "<f2",
   "scalar_bits": "3cc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1233569170581177
  },
  {
   "id": "request-09-q01-option-05",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.92578125,
   "scalar_oracle": -6.928310871124268,
   "abs_delta_scalar": 0.002529621124267578,
   "scalar_dtype": "<f2",
   "scalar_bits": "edc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0798409999115393
  },
  {
   "id": "request-09-q01-option-06",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.328125,
   "scalar_oracle": -9.343143463134766,
   "abs_delta_scalar": 0.015018463134765625,
   "scalar_dtype": "<f2",
   "scalar_bits": "aac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1172233339166269
  },
  {
   "id": "request-09-q01-option-07",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -10.5,
   "scalar_oracle": -10.518312454223633,
   "abs_delta_scalar": 0.018312454223632812,
   "scalar_dtype": "<f2",
   "scalar_bits": "40c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1308824999723583
  },
  {
   "id": "request-09-q01-option-08",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -10.59375,
   "scalar_oracle": -10.605207443237305,
   "abs_delta_scalar": 0.011457443237304688,
   "scalar_dtype": "<f2",
   "scalar_bits": "4cc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1231460829731077
  },
  {
   "id": "request-09-q01-option-09",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -7.95703125,
   "scalar_oracle": -7.95442008972168,
   "abs_delta_scalar": 0.0026111602783203125,
   "scalar_dtype": "<f2",
   "scalar_bits": "f5c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1238371669314802
  },
  {
   "id": "request-09-q01-option-10",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.578125,
   "scalar_oracle": -8.586217880249023,
   "abs_delta_scalar": 0.008092880249023438,
   "scalar_dtype": "<f2",
   "scalar_bits": "4ac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1191349580185488
  },
  {
   "id": "request-09-q02-option-00",
   "request_id": "request-09",
   "question_id": "request-09-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": -4.53125,
   "scalar_oracle": -4.542425155639648,
   "abs_delta_scalar": 0.011175155639648438,
   "scalar_dtype": "<f2",
   "scalar_bits": "88c4",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0988659160211682
  },
  {
   "id": "request-09-q02-option-01",
   "request_id": "request-09",
   "question_id": "request-09-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": 2.421875,
   "scalar_oracle": 2.413087844848633,
   "abs_delta_scalar": 0.008787155151367188,
   "scalar_dtype": "<f2",
   "scalar_bits": "d840",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.091352374991402
  },
  {
   "id": "request-09-q03-option-00",
   "request_id": "request-09",
   "question_id": "request-09-q03",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -6.79296875,
   "scalar_oracle": -6.80530309677124,
   "abs_delta_scalar": 0.012334346771240234,
   "scalar_dtype": "<f2",
   "scalar_bits": "cbc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1396339170169085
  },
  {
   "id": "request-09-q03-option-01",
   "request_id": "request-09",
   "question_id": "request-09-q03",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": 2.158203125,
   "scalar_oracle": 2.1344544887542725,
   "abs_delta_scalar": 0.02374863624572754,
   "scalar_dtype": "<f2",
   "scalar_bits": "5140",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1514510419219732
  },
  {
   "id": "request-10-q01-option-00",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 0.86767578125,
   "scalar_oracle": 0.8400804996490479,
   "abs_delta_scalar": 0.02759528160095215,
   "scalar_dtype": "<f2",
   "scalar_bits": "f13a",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.788700124947354
  },
  {
   "id": "request-10-q01-option-01",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.265625,
   "scalar_oracle": -9.282438278198242,
   "abs_delta_scalar": 0.016813278198242188,
   "scalar_dtype": "<f2",
   "scalar_bits": "a2c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0955504999728873
  },
  {
   "id": "request-10-q01-option-02",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.3515625,
   "scalar_oracle": -9.375736236572266,
   "abs_delta_scalar": 0.024173736572265625,
   "scalar_dtype": "<f2",
   "scalar_bits": "adc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.092886083992198
  },
  {
   "id": "request-10-q01-option-03",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -11.4140625,
   "scalar_oracle": -11.433075904846191,
   "abs_delta_scalar": 0.019013404846191406,
   "scalar_dtype": "<f2",
   "scalar_bits": "b5c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0828660420374945
  },
  {
   "id": "request-10-q01-option-04",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -11.2734375,
   "scalar_oracle": -11.280050277709961,
   "abs_delta_scalar": 0.0066127777099609375,
   "scalar_dtype": "<f2",
   "scalar_bits": "a3c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1055408749962226
  },
  {
   "id": "request-10-q01-option-05",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -11.75,
   "scalar_oracle": -11.762323379516602,
   "abs_delta_scalar": 0.012323379516601562,
   "scalar_dtype": "<f2",
   "scalar_bits": "e0c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0869484579889104
  },
  {
   "id": "request-10-q01-option-06",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -11.5625,
   "scalar_oracle": -11.585577011108398,
   "abs_delta_scalar": 0.023077011108398438,
   "scalar_dtype": "<f2",
   "scalar_bits": "c8c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1172107920283452
  },
  {
   "id": "request-10-q01-option-07",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -11.3046875,
   "scalar_oracle": -11.320062637329102,
   "abs_delta_scalar": 0.015375137329101562,
   "scalar_dtype": "<f2",
   "scalar_bits": "a7c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0754976249299943
  },
  {
   "id": "request-10-q01-option-08",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.44921875,
   "scalar_oracle": -7.437173843383789,
   "abs_delta_scalar": 0.012044906616210938,
   "scalar_dtype": "<f2",
   "scalar_bits": "73c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.088114875019528
  },
  {
   "id": "request-10-q01-option-09",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.75,
   "scalar_oracle": -8.777253150939941,
   "abs_delta_scalar": 0.027253150939941406,
   "scalar_dtype": "<f2",
   "scalar_bits": "60c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0758343329653144
  },
  {
   "id": "request-10-q01-option-10",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -10.1875,
   "scalar_oracle": -10.206853866577148,
   "abs_delta_scalar": 0.019353866577148438,
   "scalar_dtype": "<f2",
   "scalar_bits": "18c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.126802874961868
  },
  {
   "id": "request-10-q01-option-11",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -11.390625,
   "scalar_oracle": -11.40426254272461,
   "abs_delta_scalar": 0.013637542724609375,
   "scalar_dtype": "<f2",
   "scalar_bits": "b2c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1043115839129314
  },
  {
   "id": "request-10-q02-option-00",
   "request_id": "request-10",
   "question_id": "request-10-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": 2.751953125,
   "scalar_oracle": 2.7351107597351074,
   "abs_delta_scalar": 0.016842365264892578,
   "scalar_dtype": "<f2",
   "scalar_bits": "8141",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.079759875079617
  },
  {
   "id": "request-10-q02-option-01",
   "request_id": "request-10",
   "question_id": "request-10-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": -2.48828125,
   "scalar_oracle": -2.4982452392578125,
   "abs_delta_scalar": 0.0099639892578125,
   "scalar_dtype": "<f2",
   "scalar_bits": "fac0",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0763706249417737
  },
  {
   "id": "request-10-q03-option-00",
   "request_id": "request-10",
   "question_id": "request-10-q03",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -2.599609375,
   "scalar_oracle": -2.61518931388855,
   "abs_delta_scalar": 0.015579938888549805,
   "scalar_dtype": "<f2",
   "scalar_bits": "33c1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.9315576249500737
  },
  {
   "id": "request-10-q03-option-01",
   "request_id": "request-10",
   "question_id": "request-10-q03",
   "type": "score",
   "tokens": 31,
   "slot": 30,
   "scalar": 2.80859375,
   "scalar_oracle": 2.7923030853271484,
   "abs_delta_scalar": 0.016290664672851562,
   "scalar_dtype": "<f2",
   "scalar_bits": "9e41",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.306056415895
  },
  {
   "id": "request-11-q01-option-00",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": 5.2578125,
   "scalar_oracle": 5.246809482574463,
   "abs_delta_scalar": 0.01100301742553711,
   "scalar_dtype": "<f2",
   "scalar_bits": "4245",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1301884590648115
  },
  {
   "id": "request-11-q01-option-01",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -6.21875,
   "scalar_oracle": -6.228903293609619,
   "abs_delta_scalar": 0.01015329360961914,
   "scalar_dtype": "<f2",
   "scalar_bits": "38c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.138901749975048
  },
  {
   "id": "request-11-q01-option-02",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -7.16015625,
   "scalar_oracle": -7.158008575439453,
   "abs_delta_scalar": 0.002147674560546875,
   "scalar_dtype": "<f2",
   "scalar_bits": "29c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1359637500718236
  },
  {
   "id": "request-11-q01-option-03",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -4.40234375,
   "scalar_oracle": -4.403936386108398,
   "abs_delta_scalar": 0.0015926361083984375,
   "scalar_dtype": "<f2",
   "scalar_bits": "67c4",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1429127500159666
  },
  {
   "id": "request-11-q01-option-04",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.59375,
   "scalar_oracle": -6.593619346618652,
   "abs_delta_scalar": 0.00013065338134765625,
   "scalar_dtype": "<f2",
   "scalar_bits": "98c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.092263750033453
  },
  {
   "id": "request-11-q01-option-05",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.44140625,
   "scalar_oracle": -7.444218635559082,
   "abs_delta_scalar": 0.0028123855590820312,
   "scalar_dtype": "<f2",
   "scalar_bits": "71c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0930221669841558
  },
  {
   "id": "request-11-q01-option-06",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.1953125,
   "scalar_oracle": -9.191930770874023,
   "abs_delta_scalar": 0.0033817291259765625,
   "scalar_dtype": "<f2",
   "scalar_bits": "99c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1004795000189915
  },
  {
   "id": "request-11-q01-option-07",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.3984375,
   "scalar_oracle": -8.394148826599121,
   "abs_delta_scalar": 0.004288673400878906,
   "scalar_dtype": "<f2",
   "scalar_bits": "33c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0898940830957144
  },
  {
   "id": "request-11-q01-option-08",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.6171875,
   "scalar_oracle": -8.630069732666016,
   "abs_delta_scalar": 0.012882232666015625,
   "scalar_dtype": "<f2",
   "scalar_bits": "4fc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1275469589745626
  },
  {
   "id": "request-11-q01-option-09",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.53125,
   "scalar_oracle": -9.541899681091309,
   "abs_delta_scalar": 0.010649681091308594,
   "scalar_dtype": "<f2",
   "scalar_bits": "c4c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.142227916046977
  },
  {
   "id": "request-11-q01-option-10",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.00390625,
   "scalar_oracle": -7.013782501220703,
   "abs_delta_scalar": 0.009876251220703125,
   "scalar_dtype": "<f2",
   "scalar_bits": "01c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1003741670865566
  },
  {
   "id": "request-11-q01-option-11",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.234375,
   "scalar_oracle": -9.234328269958496,
   "abs_delta_scalar": 4.673004150390625e-05,
   "scalar_dtype": "<f2",
   "scalar_bits": "9ec8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1221102910349146
  },
  {
   "id": "request-11-q01-option-12",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.3125,
   "scalar_oracle": -9.323783874511719,
   "abs_delta_scalar": 0.01128387451171875,
   "scalar_dtype": "<f2",
   "scalar_bits": "a8c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.875134582980536
  },
  {
   "id": "request-11-q01-option-13",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.3515625,
   "scalar_oracle": -8.359192848205566,
   "abs_delta_scalar": 0.007630348205566406,
   "scalar_dtype": "<f2",
   "scalar_bits": "2dc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5656164999818429
  },
  {
   "id": "request-11-q02-option-00",
   "request_id": "request-11",
   "question_id": "request-11-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": -3.03515625,
   "scalar_oracle": -3.04154896736145,
   "abs_delta_scalar": 0.006392717361450195,
   "scalar_dtype": "<f2",
   "scalar_bits": "12c2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5412980830296874
  },
  {
   "id": "request-11-q02-option-01",
   "request_id": "request-11",
   "question_id": "request-11-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": 3.0625,
   "scalar_oracle": 3.0489227771759033,
   "abs_delta_scalar": 0.01357722282409668,
   "scalar_dtype": "<f2",
   "scalar_bits": "2042",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5456035829847679
  },
  {
   "id": "request-11-q03-option-00",
   "request_id": "request-11",
   "question_id": "request-11-q03",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -1.9833984375,
   "scalar_oracle": -1.9961133003234863,
   "abs_delta_scalar": 0.012714862823486328,
   "scalar_dtype": "<f2",
   "scalar_bits": "efbf",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5705069589894265
  },
  {
   "id": "request-11-q03-option-01",
   "request_id": "request-11",
   "question_id": "request-11-q03",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -0.7919921875,
   "scalar_oracle": -0.7945132851600647,
   "abs_delta_scalar": 0.0025210976600646973,
   "scalar_dtype": "<f2",
   "scalar_bits": "56ba",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.561302458983846
  },
  {
   "id": "request-11-q03-option-02",
   "request_id": "request-11",
   "question_id": "request-11-q03",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": 0.1658935546875,
   "scalar_oracle": 0.13405287265777588,
   "abs_delta_scalar": 0.03184068202972412,
   "scalar_dtype": "<f2",
   "scalar_bits": "4f31",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.56360124994535
  },
  {
   "id": "request-12-q01-option-00",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": 6.2578125,
   "scalar_oracle": 6.2408976554870605,
   "abs_delta_scalar": 0.016914844512939453,
   "scalar_dtype": "<f2",
   "scalar_bits": "4246",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.58425958303269
  },
  {
   "id": "request-12-q01-option-01",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -3.328125,
   "scalar_oracle": -3.3339803218841553,
   "abs_delta_scalar": 0.0058553218841552734,
   "scalar_dtype": "<f2",
   "scalar_bits": "a8c2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.584491458022967
  },
  {
   "id": "request-12-q01-option-02",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -6.76171875,
   "scalar_oracle": -6.770749568939209,
   "abs_delta_scalar": 0.009030818939208984,
   "scalar_dtype": "<f2",
   "scalar_bits": "c3c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5617916249902919
  },
  {
   "id": "request-12-q01-option-03",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -7.01171875,
   "scalar_oracle": -7.012792587280273,
   "abs_delta_scalar": 0.0010738372802734375,
   "scalar_dtype": "<f2",
   "scalar_bits": "03c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5644122080411762
  },
  {
   "id": "request-12-q01-option-04",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -7.1640625,
   "scalar_oracle": -7.17970609664917,
   "abs_delta_scalar": 0.015643596649169922,
   "scalar_dtype": "<f2",
   "scalar_bits": "2ac7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5639563329750672
  },
  {
   "id": "request-12-q01-option-05",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -7.4375,
   "scalar_oracle": -7.4471611976623535,
   "abs_delta_scalar": 0.009661197662353516,
   "scalar_dtype": "<f2",
   "scalar_bits": "70c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5847433329327032
  },
  {
   "id": "request-12-q01-option-06",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -7.15625,
   "scalar_oracle": -7.164470195770264,
   "abs_delta_scalar": 0.008220195770263672,
   "scalar_dtype": "<f2",
   "scalar_bits": "28c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5824501669267192
  },
  {
   "id": "request-12-q01-option-07",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -2.65234375,
   "scalar_oracle": -2.6470611095428467,
   "abs_delta_scalar": 0.00528264045715332,
   "scalar_dtype": "<f2",
   "scalar_bits": "4ec1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.8583250830415636
  },
  {
   "id": "request-12-q01-option-08",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.6015625,
   "scalar_oracle": -6.595610618591309,
   "abs_delta_scalar": 0.005951881408691406,
   "scalar_dtype": "<f2",
   "scalar_bits": "9ac6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5860882500419393
  },
  {
   "id": "request-12-q01-option-09",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -7.66796875,
   "scalar_oracle": -7.668771743774414,
   "abs_delta_scalar": 0.0008029937744140625,
   "scalar_dtype": "<f2",
   "scalar_bits": "abc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5644256670493633
  },
  {
   "id": "request-12-q01-option-10",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -8.25,
   "scalar_oracle": -8.25868034362793,
   "abs_delta_scalar": 0.008680343627929688,
   "scalar_dtype": "<f2",
   "scalar_bits": "20c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.584154250100255
  },
  {
   "id": "request-12-q01-option-11",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -4.796875,
   "scalar_oracle": -4.801041126251221,
   "abs_delta_scalar": 0.004166126251220703,
   "scalar_dtype": "<f2",
   "scalar_bits": "ccc4",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5823012919863686
  },
  {
   "id": "request-12-q01-option-12",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -8.078125,
   "scalar_oracle": -8.084135055541992,
   "abs_delta_scalar": 0.0060100555419921875,
   "scalar_dtype": "<f2",
   "scalar_bits": "0ac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5877028329996392
  },
  {
   "id": "request-12-q01-option-13",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -7.00390625,
   "scalar_oracle": -7.016900062561035,
   "abs_delta_scalar": 0.012993812561035156,
   "scalar_dtype": "<f2",
   "scalar_bits": "01c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5895749579649419
  },
  {
   "id": "request-12-q01-option-14",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -2.9453125,
   "scalar_oracle": -2.95692777633667,
   "abs_delta_scalar": 0.011615276336669922,
   "scalar_dtype": "<f2",
   "scalar_bits": "e4c1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5815609999699518
  },
  {
   "id": "request-12-q01-option-15",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -3.22265625,
   "scalar_oracle": -3.219796895980835,
   "abs_delta_scalar": 0.002859354019165039,
   "scalar_dtype": "<f2",
   "scalar_bits": "72c2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5798843330703676
  },
  {
   "id": "request-12-q02-option-00",
   "request_id": "request-12",
   "question_id": "request-12-q02",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": 3.052734375,
   "scalar_oracle": 3.03344464302063,
   "abs_delta_scalar": 0.019289731979370117,
   "scalar_dtype": "<f2",
   "scalar_bits": "1b42",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5825826249783859
  },
  {
   "id": "request-12-q02-option-01",
   "request_id": "request-12",
   "question_id": "request-12-q02",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": -2.224609375,
   "scalar_oracle": -2.245439291000366,
   "abs_delta_scalar": 0.02082991600036621,
   "scalar_dtype": "<f2",
   "scalar_bits": "73c0",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5834081249777228
  },
  {
   "id": "request-12-q03-option-00",
   "request_id": "request-12",
   "question_id": "request-12-q03",
   "type": "score",
   "tokens": 28,
   "slot": 27,
   "scalar": -2.693359375,
   "scalar_oracle": -2.706667423248291,
   "abs_delta_scalar": 0.013308048248291016,
   "scalar_dtype": "<f2",
   "scalar_bits": "63c1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.581117375055328
  },
  {
   "id": "request-12-q03-option-01",
   "request_id": "request-12",
   "question_id": "request-12-q03",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -1.3095703125,
   "scalar_oracle": -1.3185935020446777,
   "abs_delta_scalar": 0.009023189544677734,
   "scalar_dtype": "<f2",
   "scalar_bits": "3dbd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6188448750181124
  },
  {
   "id": "request-12-q03-option-02",
   "request_id": "request-12",
   "question_id": "request-12-q03",
   "type": "score",
   "tokens": 28,
   "slot": 27,
   "scalar": 1.52734375,
   "scalar_oracle": 1.510910987854004,
   "abs_delta_scalar": 0.016432762145996094,
   "scalar_dtype": "<f2",
   "scalar_bits": "1c3e",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5809840000001714
  },
  {
   "id": "request-13-q01-option-00",
   "request_id": "request-13",
   "question_id": "request-13-q01",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": 4.42578125,
   "scalar_oracle": 4.405121803283691,
   "abs_delta_scalar": 0.020659446716308594,
   "scalar_dtype": "<f2",
   "scalar_bits": "6d44",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.9885026250267401
  },
  {
   "id": "request-13-q01-option-01",
   "request_id": "request-13",
   "question_id": "request-13-q01",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -9.5859375,
   "scalar_oracle": -9.618844985961914,
   "abs_delta_scalar": 0.03290748596191406,
   "scalar_dtype": "<f2",
   "scalar_bits": "cbc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6406029999488965
  },
  {
   "id": "request-13-q02-option-00",
   "request_id": "request-13",
   "question_id": "request-13-q02",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": 5.39453125,
   "scalar_oracle": 5.3750176429748535,
   "abs_delta_scalar": 0.019513607025146484,
   "scalar_dtype": "<f2",
   "scalar_bits": "6545",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6376622089883313
  },
  {
   "id": "request-13-q02-option-01",
   "request_id": "request-13",
   "question_id": "request-13-q02",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -10.21875,
   "scalar_oracle": -10.238785743713379,
   "abs_delta_scalar": 0.020035743713378906,
   "scalar_dtype": "<f2",
   "scalar_bits": "1cc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6428456249414012
  },
  {
   "id": "request-13-q03-option-00",
   "request_id": "request-13",
   "question_id": "request-13-q03",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": 7.06640625,
   "scalar_oracle": 7.067084312438965,
   "abs_delta_scalar": 0.0006780624389648438,
   "scalar_dtype": "<f2",
   "scalar_bits": "1147",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.639042541035451
  },
  {
   "id": "request-13-q03-option-01",
   "request_id": "request-13",
   "question_id": "request-13-q03",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -10.7890625,
   "scalar_oracle": -10.796977996826172,
   "abs_delta_scalar": 0.007915496826171875,
   "scalar_dtype": "<f2",
   "scalar_bits": "65c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6391556250164285
  },
  {
   "id": "request-13-q04-option-00",
   "request_id": "request-13",
   "question_id": "request-13-q04",
   "type": "noul",
   "tokens": 29,
   "slot": 28,
   "scalar": 4.60546875,
   "scalar_oracle": 4.599853515625,
   "abs_delta_scalar": 0.005615234375,
   "scalar_dtype": "<f2",
   "scalar_bits": "9b44",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6030310000060126
  },
  {
   "id": "request-13-q04-option-01",
   "request_id": "request-13",
   "question_id": "request-13-q04",
   "type": "noul",
   "tokens": 29,
   "slot": 28,
   "scalar": -1.0859375,
   "scalar_oracle": -1.0872465372085571,
   "abs_delta_scalar": 0.001309037208557129,
   "scalar_dtype": "<f2",
   "scalar_bits": "58bc",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6014110830146819
  },
  {
   "id": "request-13-q05-option-00",
   "request_id": "request-13",
   "question_id": "request-13-q05",
   "type": "score",
   "tokens": 31,
   "slot": 30,
   "scalar": 1.6240234375,
   "scalar_oracle": 1.6196714639663696,
   "abs_delta_scalar": 0.004351973533630371,
   "scalar_dtype": "<f2",
   "scalar_bits": "7f3e",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6365804999368265
  },
  {
   "id": "request-13-q05-option-01",
   "request_id": "request-13",
   "question_id": "request-13-q05",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": 0.2861328125,
   "scalar_oracle": 0.27900874614715576,
   "abs_delta_scalar": 0.007124066352844238,
   "scalar_dtype": "<f2",
   "scalar_bits": "9434",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6185325420228764
  },
  {
   "id": "request-13-q05-option-02",
   "request_id": "request-13",
   "question_id": "request-13-q05",
   "type": "score",
   "tokens": 31,
   "slot": 30,
   "scalar": -1.291015625,
   "scalar_oracle": -1.3120677471160889,
   "abs_delta_scalar": 0.021052122116088867,
   "scalar_dtype": "<f2",
   "scalar_bits": "2abd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6377986249281093
  },
  {
   "id": "request-13-q05-option-03",
   "request_id": "request-13",
   "question_id": "request-13-q05",
   "type": "score",
   "tokens": 31,
   "slot": 30,
   "scalar": -3.94921875,
   "scalar_oracle": -3.97291898727417,
   "abs_delta_scalar": 0.023700237274169922,
   "scalar_dtype": "<f2",
   "scalar_bits": "e6c3",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6372187499655411
  },
  {
   "id": "request-14-q01-option-00",
   "request_id": "request-14",
   "question_id": "request-14-q01",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": 3.595703125,
   "scalar_oracle": 3.573038101196289,
   "abs_delta_scalar": 0.022665023803710938,
   "scalar_dtype": "<f2",
   "scalar_bits": "3143",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6416386669734493
  },
  {
   "id": "request-14-q01-option-01",
   "request_id": "request-14",
   "question_id": "request-14-q01",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -10.8828125,
   "scalar_oracle": -10.914566040039062,
   "abs_delta_scalar": 0.0317535400390625,
   "scalar_dtype": "<f2",
   "scalar_bits": "71c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6396236669970676
  },
  {
   "id": "request-14-q02-option-00",
   "request_id": "request-14",
   "question_id": "request-14-q02",
   "type": "choice",
   "tokens": 32,
   "slot": 31,
   "scalar": -0.386962890625,
   "scalar_oracle": -0.42073899507522583,
   "abs_delta_scalar": 0.03377610445022583,
   "scalar_dtype": "<f2",
   "scalar_bits": "31b6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.197985874954611
  },
  {
   "id": "request-14-q02-option-01",
   "request_id": "request-14",
   "question_id": "request-14-q02",
   "type": "choice",
   "tokens": 32,
   "slot": 31,
   "scalar": -9.7265625,
   "scalar_oracle": -9.746484756469727,
   "abs_delta_scalar": 0.019922256469726562,
   "scalar_dtype": "<f2",
   "scalar_bits": "ddc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6708575839875266
  },
  {
   "id": "request-14-q03-option-00",
   "request_id": "request-14",
   "question_id": "request-14-q03",
   "type": "choice",
   "tokens": 32,
   "slot": 31,
   "scalar": 2.4296875,
   "scalar_oracle": 2.417165756225586,
   "abs_delta_scalar": 0.012521743774414062,
   "scalar_dtype": "<f2",
   "scalar_bits": "dc40",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6692924579838291
  },
  {
   "id": "request-14-q03-option-01",
   "request_id": "request-14",
   "question_id": "request-14-q03",
   "type": "choice",
   "tokens": 32,
   "slot": 31,
   "scalar": -10.453125,
   "scalar_oracle": -10.482742309570312,
   "abs_delta_scalar": 0.0296173095703125,
   "scalar_dtype": "<f2",
   "scalar_bits": "3ac9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6690292080165818
  },
  {
   "id": "request-14-q04-option-00",
   "request_id": "request-14",
   "question_id": "request-14-q04",
   "type": "noul",
   "tokens": 30,
   "slot": 29,
   "scalar": -1.5673828125,
   "scalar_oracle": -1.5531220436096191,
   "abs_delta_scalar": 0.01426076889038086,
   "scalar_dtype": "<f2",
   "scalar_bits": "45be",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6327583329984918
  },
  {
   "id": "request-14-q04-option-01",
   "request_id": "request-14",
   "question_id": "request-14-q04",
   "type": "noul",
   "tokens": 30,
   "slot": 29,
   "scalar": 1.375,
   "scalar_oracle": 1.3508743047714233,
   "abs_delta_scalar": 0.02412569522857666,
   "scalar_dtype": "<f2",
   "scalar_bits": "803d",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6290322500281036
  },
  {
   "id": "request-14-q05-option-00",
   "request_id": "request-14",
   "question_id": "request-14-q05",
   "type": "score",
   "tokens": 29,
   "slot": 28,
   "scalar": -7.94921875,
   "scalar_oracle": -7.972799301147461,
   "abs_delta_scalar": 0.023580551147460938,
   "scalar_dtype": "<f2",
   "scalar_bits": "f3c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6083601249847561
  },
  {
   "id": "request-14-q05-option-01",
   "request_id": "request-14",
   "question_id": "request-14-q05",
   "type": "score",
   "tokens": 28,
   "slot": 27,
   "scalar": -3.15234375,
   "scalar_oracle": -3.1746554374694824,
   "abs_delta_scalar": 0.022311687469482422,
   "scalar_dtype": "<f2",
   "scalar_bits": "4ec2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5896105420542881
  },
  {
   "id": "request-14-q05-option-02",
   "request_id": "request-14",
   "question_id": "request-14-q05",
   "type": "score",
   "tokens": 28,
   "slot": 27,
   "scalar": 0.01216888427734375,
   "scalar_oracle": -0.007742464542388916,
   "abs_delta_scalar": 0.019911348819732666,
   "scalar_dtype": "<f2",
   "scalar_bits": "3b22",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5910795410163701
  },
  {
   "id": "request-14-q05-option-03",
   "request_id": "request-14",
   "question_id": "request-14-q05",
   "type": "score",
   "tokens": 29,
   "slot": 28,
   "scalar": -1.2724609375,
   "scalar_oracle": -1.2937160730361938,
   "abs_delta_scalar": 0.021255135536193848,
   "scalar_dtype": "<f2",
   "scalar_bits": "17bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6119194589555264
  },
  {
   "id": "request-15-q01-option-00",
   "request_id": "request-15",
   "question_id": "request-15-q01",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -0.650390625,
   "scalar_oracle": -0.6910357475280762,
   "abs_delta_scalar": 0.04064512252807617,
   "scalar_dtype": "<f2",
   "scalar_bits": "34b9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6517045419896021
  },
  {
   "id": "request-15-q01-option-01",
   "request_id": "request-15",
   "question_id": "request-15-q01",
   "type": "choice",
   "tokens": 30,
   "slot": 29,
   "scalar": -14.578125,
   "scalar_oracle": -14.593600273132324,
   "abs_delta_scalar": 0.015475273132324219,
   "scalar_dtype": "<f2",
   "scalar_bits": "4acb",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6283270840067416
  },
  {
   "id": "request-15-q02-option-00",
   "request_id": "request-15",
   "question_id": "request-15-q02",
   "type": "choice",
   "tokens": 30,
   "slot": 29,
   "scalar": 6.671875,
   "scalar_oracle": 6.661491870880127,
   "abs_delta_scalar": 0.010383129119873047,
   "scalar_dtype": "<f2",
   "scalar_bits": "ac46",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6323016670066863
  },
  {
   "id": "request-15-q02-option-01",
   "request_id": "request-15",
   "question_id": "request-15-q02",
   "type": "choice",
   "tokens": 30,
   "slot": 29,
   "scalar": -14.5625,
   "scalar_oracle": -14.577804565429688,
   "abs_delta_scalar": 0.0153045654296875,
   "scalar_dtype": "<f2",
   "scalar_bits": "48cb",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6271793749183416
  },
  {
   "id": "request-15-q03-option-00",
   "request_id": "request-15",
   "question_id": "request-15-q03",
   "type": "choice",
   "tokens": 30,
   "slot": 29,
   "scalar": -2.009765625,
   "scalar_oracle": -2.031851053237915,
   "abs_delta_scalar": 0.02208542823791504,
   "scalar_dtype": "<f2",
   "scalar_bits": "05c0",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 25.168386582983658
  },
  {
   "id": "request-15-q03-option-01",
   "request_id": "request-15",
   "question_id": "request-15-q03",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -10.9921875,
   "scalar_oracle": -11.027084350585938,
   "abs_delta_scalar": 0.0348968505859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "7fc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6536101249512285
  },
  {
   "id": "request-15-q04-option-00",
   "request_id": "request-15",
   "question_id": "request-15-q04",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": -1.71875,
   "scalar_oracle": -1.7337946891784668,
   "abs_delta_scalar": 0.015044689178466797,
   "scalar_dtype": "<f2",
   "scalar_bits": "e0be",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5860293749719858
  },
  {
   "id": "request-15-q04-option-01",
   "request_id": "request-15",
   "question_id": "request-15-q04",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": 1.373046875,
   "scalar_oracle": 1.3656667470932007,
   "abs_delta_scalar": 0.007380127906799316,
   "scalar_dtype": "<f2",
   "scalar_bits": "7e3d",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5887942919507623
  },
  {
   "id": "request-15-q05-option-00",
   "request_id": "request-15",
   "question_id": "request-15-q05",
   "type": "score",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.4296875,
   "scalar_oracle": -6.44263219833374,
   "abs_delta_scalar": 0.012944698333740234,
   "scalar_dtype": "<f2",
   "scalar_bits": "6ec6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5882069170475006
  },
  {
   "id": "request-15-q05-option-01",
   "request_id": "request-15",
   "question_id": "request-15-q05",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -2.8125,
   "scalar_oracle": -2.8297576904296875,
   "abs_delta_scalar": 0.0172576904296875,
   "scalar_dtype": "<f2",
   "scalar_bits": "a0c1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5665512080304325
  },
  {
   "id": "request-15-q05-option-02",
   "request_id": "request-15",
   "question_id": "request-15-q05",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -0.83984375,
   "scalar_oracle": -0.8664757013320923,
   "abs_delta_scalar": 0.026631951332092285,
   "scalar_dtype": "<f2",
   "scalar_bits": "b8ba",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5652286250842735
  },
  {
   "id": "request-15-q05-option-03",
   "request_id": "request-15",
   "question_id": "request-15-q05",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": 1.4873046875,
   "scalar_oracle": 1.476930022239685,
   "abs_delta_scalar": 0.010374665260314941,
   "scalar_dtype": "<f2",
   "scalar_bits": "f33d",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5698203749489039
  },
  {
   "id": "request-15-q05-option-04",
   "request_id": "request-15",
   "question_id": "request-15-q05",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -0.7861328125,
   "scalar_oracle": -0.7765030860900879,
   "abs_delta_scalar": 0.00962972640991211,
   "scalar_dtype": "<f2",
   "scalar_bits": "4aba",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5691289999522269
  },
  {
   "id": "request-16-q01-option-00",
   "request_id": "request-16",
   "question_id": "request-16-q01",
   "type": "choice",
   "tokens": 35,
   "slot": 34,
   "scalar": 3.55078125,
   "scalar_oracle": 3.515467405319214,
   "abs_delta_scalar": 0.03531384468078613,
   "scalar_dtype": "<f2",
   "scalar_bits": "1a43",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7325290420558304
  },
  {
   "id": "request-16-q01-option-01",
   "request_id": "request-16",
   "question_id": "request-16-q01",
   "type": "choice",
   "tokens": 34,
   "slot": 33,
   "scalar": -8.203125,
   "scalar_oracle": -8.229701042175293,
   "abs_delta_scalar": 0.02657604217529297,
   "scalar_dtype": "<f2",
   "scalar_bits": "1ac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7107370829908177
  },
  {
   "id": "request-16-q02-option-00",
   "request_id": "request-16",
   "question_id": "request-16-q02",
   "type": "choice",
   "tokens": 33,
   "slot": 32,
   "scalar": 3.33203125,
   "scalar_oracle": 3.330043077468872,
   "abs_delta_scalar": 0.0019881725311279297,
   "scalar_dtype": "<f2",
   "scalar_bits": "aa42",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.689332416979596
  },
  {
   "id": "request-16-q02-option-01",
   "request_id": "request-16",
   "question_id": "request-16-q02",
   "type": "choice",
   "tokens": 33,
   "slot": 32,
   "scalar": -5.1171875,
   "scalar_oracle": -5.142807960510254,
   "abs_delta_scalar": 0.025620460510253906,
   "scalar_dtype": "<f2",
   "scalar_bits": "1ec5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6916697500273585
  },
  {
   "id": "request-16-q03-option-00",
   "request_id": "request-16",
   "question_id": "request-16-q03",
   "type": "choice",
   "tokens": 34,
   "slot": 33,
   "scalar": -0.66943359375,
   "scalar_oracle": -0.6904910802841187,
   "abs_delta_scalar": 0.021057486534118652,
   "scalar_dtype": "<f2",
   "scalar_bits": "5bb9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7114879590226337
  },
  {
   "id": "request-16-q03-option-01",
   "request_id": "request-16",
   "question_id": "request-16-q03",
   "type": "choice",
   "tokens": 34,
   "slot": 33,
   "scalar": -13.1875,
   "scalar_oracle": -13.2025728225708,
   "abs_delta_scalar": 0.015072822570800781,
   "scalar_dtype": "<f2",
   "scalar_bits": "98ca",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 3.0480251249391586
  },
  {
   "id": "request-16-q04-option-00",
   "request_id": "request-16",
   "question_id": "request-16-q04",
   "type": "noul",
   "tokens": 32,
   "slot": 31,
   "scalar": 2.2109375,
   "scalar_oracle": 2.1954517364501953,
   "abs_delta_scalar": 0.015485763549804688,
   "scalar_dtype": "<f2",
   "scalar_bits": "6c40",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6690505830338225
  },
  {
   "id": "request-16-q04-option-01",
   "request_id": "request-16",
   "question_id": "request-16-q04",
   "type": "noul",
   "tokens": 32,
   "slot": 31,
   "scalar": -2.6953125,
   "scalar_oracle": -2.713536024093628,
   "abs_delta_scalar": 0.01822352409362793,
   "scalar_dtype": "<f2",
   "scalar_bits": "64c1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.671406332985498
  },
  {
   "id": "request-16-q05-option-00",
   "request_id": "request-16",
   "question_id": "request-16-q05",
   "type": "score",
   "tokens": 31,
   "slot": 30,
   "scalar": -2.46875,
   "scalar_oracle": -2.494438886642456,
   "abs_delta_scalar": 0.025688886642456055,
   "scalar_dtype": "<f2",
   "scalar_bits": "f0c0",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6476196669973433
  },
  {
   "id": "request-16-q05-option-01",
   "request_id": "request-16",
   "question_id": "request-16-q05",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -0.1781005859375,
   "scalar_oracle": -0.20091819763183594,
   "abs_delta_scalar": 0.022817611694335938,
   "scalar_dtype": "<f2",
   "scalar_bits": "b3b1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6751692920224741
  },
  {
   "id": "request-16-q05-option-02",
   "request_id": "request-16",
   "question_id": "request-16-q05",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -1.2763671875,
   "scalar_oracle": -1.2986819744110107,
   "abs_delta_scalar": 0.022314786911010742,
   "scalar_dtype": "<f2",
   "scalar_bits": "1bbd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6281801670556888
  },
  {
   "id": "request-16-q05-option-03",
   "request_id": "request-16",
   "question_id": "request-16-q05",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -3.671875,
   "scalar_oracle": -3.687429666519165,
   "abs_delta_scalar": 0.015554666519165039,
   "scalar_dtype": "<f2",
   "scalar_bits": "58c3",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.626823875005357
  },
  {
   "id": "request-16-q05-option-04",
   "request_id": "request-16",
   "question_id": "request-16-q05",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -7.6796875,
   "scalar_oracle": -7.6829833984375,
   "abs_delta_scalar": 0.0032958984375,
   "scalar_dtype": "<f2",
   "scalar_bits": "aec7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6278400829760358
  },
  {
   "id": "request-17-q01-option-00",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 5.5390625,
   "scalar_oracle": 5.513814926147461,
   "abs_delta_scalar": 0.025247573852539062,
   "scalar_dtype": "<f2",
   "scalar_bits": "8a45",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5468805419513956
  },
  {
   "id": "request-17-q01-option-01",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.296875,
   "scalar_oracle": -7.31196403503418,
   "abs_delta_scalar": 0.015089035034179688,
   "scalar_dtype": "<f2",
   "scalar_bits": "4cc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.550220042001456
  },
  {
   "id": "request-17-q01-option-02",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.6171875,
   "scalar_oracle": -7.631340503692627,
   "abs_delta_scalar": 0.014153003692626953,
   "scalar_dtype": "<f2",
   "scalar_bits": "9ec7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5467496250057593
  },
  {
   "id": "request-17-q01-option-03",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.90625,
   "scalar_oracle": -7.937613487243652,
   "abs_delta_scalar": 0.031363487243652344,
   "scalar_dtype": "<f2",
   "scalar_bits": "e8c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5478710419265553
  },
  {
   "id": "request-17-q01-option-04",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.3359375,
   "scalar_oracle": -8.349449157714844,
   "abs_delta_scalar": 0.01351165771484375,
   "scalar_dtype": "<f2",
   "scalar_bits": "2bc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5262382919900119
  },
  {
   "id": "request-17-q01-option-05",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.5078125,
   "scalar_oracle": -8.52861499786377,
   "abs_delta_scalar": 0.02080249786376953,
   "scalar_dtype": "<f2",
   "scalar_bits": "41c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.525932666962035
  },
  {
   "id": "request-17-q01-option-06",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.5390625,
   "scalar_oracle": -8.547520637512207,
   "abs_delta_scalar": 0.008458137512207031,
   "scalar_dtype": "<f2",
   "scalar_bits": "45c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.6372006660094485
  },
  {
   "id": "request-17-q01-option-07",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -6.640625,
   "scalar_oracle": -6.649753093719482,
   "abs_delta_scalar": 0.009128093719482422,
   "scalar_dtype": "<f2",
   "scalar_bits": "a4c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5274439170025289
  },
  {
   "id": "request-17-q01-option-08",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 2.853515625,
   "scalar_oracle": 2.8335750102996826,
   "abs_delta_scalar": 0.019940614700317383,
   "scalar_dtype": "<f2",
   "scalar_bits": "b541",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.54519024991896
  },
  {
   "id": "request-17-q01-option-09",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -5.97265625,
   "scalar_oracle": -5.990054130554199,
   "abs_delta_scalar": 0.01739788055419922,
   "scalar_dtype": "<f2",
   "scalar_bits": "f9c5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5476647500181571
  },
  {
   "id": "request-17-q01-option-10",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.578125,
   "scalar_oracle": -8.59441089630127,
   "abs_delta_scalar": 0.01628589630126953,
   "scalar_dtype": "<f2",
   "scalar_bits": "4ac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5243072079028934
  },
  {
   "id": "request-17-q01-option-11",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.8828125,
   "scalar_oracle": -8.899070739746094,
   "abs_delta_scalar": 0.01625823974609375,
   "scalar_dtype": "<f2",
   "scalar_bits": "71c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5485785830533132
  },
  {
   "id": "request-17-q01-option-12",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.8515625,
   "scalar_oracle": -8.87210464477539,
   "abs_delta_scalar": 0.020542144775390625,
   "scalar_dtype": "<f2",
   "scalar_bits": "6dc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5243492920417339
  },
  {
   "id": "request-17-q01-option-13",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.96875,
   "scalar_oracle": -9.984298706054688,
   "abs_delta_scalar": 0.0155487060546875,
   "scalar_dtype": "<f2",
   "scalar_bits": "fcc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.545190625009127
  },
  {
   "id": "request-17-q01-option-14",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.90625,
   "scalar_oracle": -9.920188903808594,
   "abs_delta_scalar": 0.01393890380859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "f4c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5452464999398217
  },
  {
   "id": "request-17-q01-option-15",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.7265625,
   "scalar_oracle": -9.742130279541016,
   "abs_delta_scalar": 0.015567779541015625,
   "scalar_dtype": "<f2",
   "scalar_bits": "ddc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5469132079742849
  },
  {
   "id": "request-17-q01-option-16",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -10.671875,
   "scalar_oracle": -10.690881729125977,
   "abs_delta_scalar": 0.019006729125976562,
   "scalar_dtype": "<f2",
   "scalar_bits": "56c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5662051249528304
  },
  {
   "id": "request-17-q01-option-17",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.46875,
   "scalar_oracle": -10.486421585083008,
   "abs_delta_scalar": 0.017671585083007812,
   "scalar_dtype": "<f2",
   "scalar_bits": "3cc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5431163340108469
  },
  {
   "id": "request-17-q01-option-18",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.171875,
   "scalar_oracle": -9.18571662902832,
   "abs_delta_scalar": 0.013841629028320312,
   "scalar_dtype": "<f2",
   "scalar_bits": "96c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6095199160045013
  },
  {
   "id": "request-17-q01-option-19",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -7.59375,
   "scalar_oracle": -7.604816436767578,
   "abs_delta_scalar": 0.011066436767578125,
   "scalar_dtype": "<f2",
   "scalar_bits": "98c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.56599395792
  },
  {
   "id": "request-18-q01-option-00",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -8.046875,
   "scalar_oracle": -8.074076652526855,
   "abs_delta_scalar": 0.02720165252685547,
   "scalar_dtype": "<f2",
   "scalar_bits": "06c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.356691375025548
  },
  {
   "id": "request-18-q01-option-01",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": 3.837890625,
   "scalar_oracle": 3.807345151901245,
   "abs_delta_scalar": 0.030545473098754883,
   "scalar_dtype": "<f2",
   "scalar_bits": "ad43",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5929245000006631
  },
  {
   "id": "request-18-q01-option-02",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.828125,
   "scalar_oracle": -6.854000091552734,
   "abs_delta_scalar": 0.025875091552734375,
   "scalar_dtype": "<f2",
   "scalar_bits": "d4c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5917403750354424
  },
  {
   "id": "request-18-q01-option-03",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -8.1640625,
   "scalar_oracle": -8.165847778320312,
   "abs_delta_scalar": 0.0017852783203125,
   "scalar_dtype": "<f2",
   "scalar_bits": "15c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.593924832995981
  },
  {
   "id": "request-18-q01-option-04",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.8984375,
   "scalar_oracle": -8.922479629516602,
   "abs_delta_scalar": 0.024042129516601562,
   "scalar_dtype": "<f2",
   "scalar_bits": "73c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.577205833978951
  },
  {
   "id": "request-18-q01-option-05",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.375,
   "scalar_oracle": -8.393821716308594,
   "abs_delta_scalar": 0.01882171630859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "30c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5801962920231745
  },
  {
   "id": "request-18-q01-option-06",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.5,
   "scalar_oracle": -6.5011138916015625,
   "abs_delta_scalar": 0.0011138916015625,
   "scalar_dtype": "<f2",
   "scalar_bits": "80c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5989397920202464
  },
  {
   "id": "request-18-q01-option-07",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -5.9921875,
   "scalar_oracle": -6.00640869140625,
   "abs_delta_scalar": 0.01422119140625,
   "scalar_dtype": "<f2",
   "scalar_bits": "fec5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5782645000144839
  },
  {
   "id": "request-18-q01-option-08",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -5.3203125,
   "scalar_oracle": -5.335151672363281,
   "abs_delta_scalar": 0.01483917236328125,
   "scalar_dtype": "<f2",
   "scalar_bits": "52c5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5928781250258908
  },
  {
   "id": "request-18-q01-option-09",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.57421875,
   "scalar_oracle": -6.5979790687561035,
   "abs_delta_scalar": 0.023760318756103516,
   "scalar_dtype": "<f2",
   "scalar_bits": "93c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8945733329746872
  },
  {
   "id": "request-18-q01-option-10",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.8671875,
   "scalar_oracle": -9.877313613891602,
   "abs_delta_scalar": 0.010126113891601562,
   "scalar_dtype": "<f2",
   "scalar_bits": "efc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1438539589289576
  },
  {
   "id": "request-18-q01-option-11",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -9.4453125,
   "scalar_oracle": -9.465591430664062,
   "abs_delta_scalar": 0.0202789306640625,
   "scalar_dtype": "<f2",
   "scalar_bits": "b9c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.184849333949387
  },
  {
   "id": "request-18-q01-option-12",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.2265625,
   "scalar_oracle": -9.240874290466309,
   "abs_delta_scalar": 0.014311790466308594,
   "scalar_dtype": "<f2",
   "scalar_bits": "9dc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1416810411028564
  },
  {
   "id": "request-18-q01-option-13",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -9.84375,
   "scalar_oracle": -9.860201835632324,
   "abs_delta_scalar": 0.01645183563232422,
   "scalar_dtype": "<f2",
   "scalar_bits": "ecc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1897086669923738
  },
  {
   "id": "request-18-q01-option-14",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -9.6640625,
   "scalar_oracle": -9.681700706481934,
   "abs_delta_scalar": 0.017638206481933594,
   "scalar_dtype": "<f2",
   "scalar_bits": "d5c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 3.056918208952993
  },
  {
   "id": "request-18-q01-option-15",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -10.7890625,
   "scalar_oracle": -10.806360244750977,
   "abs_delta_scalar": 0.017297744750976562,
   "scalar_dtype": "<f2",
   "scalar_bits": "65c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.193139374954626
  },
  {
   "id": "request-18-q01-option-16",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 29,
   "slot": 28,
   "scalar": -11.6953125,
   "scalar_oracle": -11.715137481689453,
   "abs_delta_scalar": 0.019824981689453125,
   "scalar_dtype": "<f2",
   "scalar_bits": "d9c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.237531334045343
  },
  {
   "id": "request-18-q01-option-17",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -10.6328125,
   "scalar_oracle": -10.642301559448242,
   "abs_delta_scalar": 0.009489059448242188,
   "scalar_dtype": "<f2",
   "scalar_bits": "51c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1833106249105185
  },
  {
   "id": "request-18-q01-option-18",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -3.943359375,
   "scalar_oracle": -3.9700522422790527,
   "abs_delta_scalar": 0.026692867279052734,
   "scalar_dtype": "<f2",
   "scalar_bits": "e3c3",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.2097473751055077
  },
  {
   "id": "request-18-q01-option-19",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": 1.884765625,
   "scalar_oracle": 1.8557240962982178,
   "abs_delta_scalar": 0.029041528701782227,
   "scalar_dtype": "<f2",
   "scalar_bits": "8a3f",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1473003340652213
  },
  {
   "id": "request-18-q01-option-20",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -5.1796875,
   "scalar_oracle": -5.19008207321167,
   "abs_delta_scalar": 0.010394573211669922,
   "scalar_dtype": "<f2",
   "scalar_bits": "2ec5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.178713500034064
  },
  {
   "id": "request-18-q01-option-21",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.12890625,
   "scalar_oracle": -6.140894889831543,
   "abs_delta_scalar": 0.011988639831542969,
   "scalar_dtype": "<f2",
   "scalar_bits": "21c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.179977708030492
  },
  {
   "id": "request-18-q01-option-22",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.7578125,
   "scalar_oracle": -6.777218341827393,
   "abs_delta_scalar": 0.019405841827392578,
   "scalar_dtype": "<f2",
   "scalar_bits": "c2c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1939260830404237
  },
  {
   "id": "request-18-q01-option-23",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.1640625,
   "scalar_oracle": -8.173320770263672,
   "abs_delta_scalar": 0.009258270263671875,
   "scalar_dtype": "<f2",
   "scalar_bits": "15c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1317083329195157
  },
  {
   "id": "request-19-q01-option-00",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.609375,
   "scalar_oracle": -7.635459899902344,
   "abs_delta_scalar": 0.02608489990234375,
   "scalar_dtype": "<f2",
   "scalar_bits": "9cc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1106644169194624
  },
  {
   "id": "request-19-q01-option-01",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.2578125,
   "scalar_oracle": -8.273568153381348,
   "abs_delta_scalar": 0.015755653381347656,
   "scalar_dtype": "<f2",
   "scalar_bits": "21c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.110960332909599
  },
  {
   "id": "request-19-q01-option-02",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.6015625,
   "scalar_oracle": -7.611283779144287,
   "abs_delta_scalar": 0.00972127914428711,
   "scalar_dtype": "<f2",
   "scalar_bits": "9ac7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1083490420132875
  },
  {
   "id": "request-19-q01-option-03",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -1.0126953125,
   "scalar_oracle": -1.0317230224609375,
   "abs_delta_scalar": 0.0190277099609375,
   "scalar_dtype": "<f2",
   "scalar_bits": "0dbc",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1102399579249322
  },
  {
   "id": "request-19-q01-option-04",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -9.1953125,
   "scalar_oracle": -9.20922565460205,
   "abs_delta_scalar": 0.013913154602050781,
   "scalar_dtype": "<f2",
   "scalar_bits": "99c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.640929584042169
  },
  {
   "id": "request-19-q01-option-05",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.703125,
   "scalar_oracle": -8.708393096923828,
   "abs_delta_scalar": 0.005268096923828125,
   "scalar_dtype": "<f2",
   "scalar_bits": "5ac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0577162079280242
  },
  {
   "id": "request-19-q01-option-06",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.09375,
   "scalar_oracle": -10.114972114562988,
   "abs_delta_scalar": 0.02122211456298828,
   "scalar_dtype": "<f2",
   "scalar_bits": "0cc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0989559579174966
  },
  {
   "id": "request-19-q01-option-07",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -9.6796875,
   "scalar_oracle": -9.697352409362793,
   "abs_delta_scalar": 0.01766490936279297,
   "scalar_dtype": "<f2",
   "scalar_bits": "d7c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0527168749831617
  },
  {
   "id": "request-19-q01-option-08",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -5.94921875,
   "scalar_oracle": -5.983921051025391,
   "abs_delta_scalar": 0.034702301025390625,
   "scalar_dtype": "<f2",
   "scalar_bits": "f3c5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0990629160078242
  },
  {
   "id": "request-19-q01-option-09",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.609375,
   "scalar_oracle": -9.618802070617676,
   "abs_delta_scalar": 0.009427070617675781,
   "scalar_dtype": "<f2",
   "scalar_bits": "cec8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1194477919489145
  },
  {
   "id": "request-19-q01-option-10",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -10.5078125,
   "scalar_oracle": -10.518702507019043,
   "abs_delta_scalar": 0.010890007019042969,
   "scalar_dtype": "<f2",
   "scalar_bits": "41c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.079010291956365
  },
  {
   "id": "request-19-q01-option-11",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.109375,
   "scalar_oracle": -10.130147933959961,
   "abs_delta_scalar": 0.020772933959960938,
   "scalar_dtype": "<f2",
   "scalar_bits": "0ec9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0811809579608962
  },
  {
   "id": "request-19-q01-option-12",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.015625,
   "scalar_oracle": -8.015636444091797,
   "abs_delta_scalar": 1.1444091796875e-05,
   "scalar_dtype": "<f2",
   "scalar_bits": "02c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0896496248897165
  },
  {
   "id": "request-19-q01-option-13",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.9296875,
   "scalar_oracle": -9.947691917419434,
   "abs_delta_scalar": 0.018004417419433594,
   "scalar_dtype": "<f2",
   "scalar_bits": "f7c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1145908329635859
  },
  {
   "id": "request-19-q01-option-14",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.59375,
   "scalar_oracle": -10.613463401794434,
   "abs_delta_scalar": 0.019713401794433594,
   "scalar_dtype": "<f2",
   "scalar_bits": "4cc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.110769166960381
  },
  {
   "id": "request-19-q01-option-15",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.671875,
   "scalar_oracle": -9.693241119384766,
   "abs_delta_scalar": 0.021366119384765625,
   "scalar_dtype": "<f2",
   "scalar_bits": "d6c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1068090840708464
  },
  {
   "id": "request-19-q01-option-16",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -11.84375,
   "scalar_oracle": -11.859024047851562,
   "abs_delta_scalar": 0.0152740478515625,
   "scalar_dtype": "<f2",
   "scalar_bits": "ecc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1449819171102718
  },
  {
   "id": "request-19-q01-option-17",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -11.140625,
   "scalar_oracle": -11.160334587097168,
   "abs_delta_scalar": 0.01970958709716797,
   "scalar_dtype": "<f2",
   "scalar_bits": "92c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1244045000057667
  },
  {
   "id": "request-19-q01-option-18",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.375,
   "scalar_oracle": -9.379319190979004,
   "abs_delta_scalar": 0.004319190979003906,
   "scalar_dtype": "<f2",
   "scalar_bits": "b0c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 4.446834499947727
  },
  {
   "id": "request-19-q01-option-19",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.1953125,
   "scalar_oracle": -8.208106994628906,
   "abs_delta_scalar": 0.01279449462890625,
   "scalar_dtype": "<f2",
   "scalar_bits": "19c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0557246670359746
  },
  {
   "id": "request-19-q01-option-20",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.9140625,
   "scalar_oracle": -8.927600860595703,
   "abs_delta_scalar": 0.013538360595703125,
   "scalar_dtype": "<f2",
   "scalar_bits": "75c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0804123749257997
  },
  {
   "id": "request-19-q01-option-21",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.99609375,
   "scalar_oracle": -7.999220848083496,
   "abs_delta_scalar": 0.0031270980834960938,
   "scalar_dtype": "<f2",
   "scalar_bits": "ffc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0904507079394534
  },
  {
   "id": "request-19-q01-option-22",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.95703125,
   "scalar_oracle": -6.959987640380859,
   "abs_delta_scalar": 0.002956390380859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "f5c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0709136250661686
  },
  {
   "id": "request-19-q01-option-23",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": 1.2744140625,
   "scalar_oracle": 1.2542405128479004,
   "abs_delta_scalar": 0.02017354965209961,
   "scalar_dtype": "<f2",
   "scalar_bits": "193d",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.050384208909236
  },
  {
   "id": "request-19-q01-option-24",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -7.11328125,
   "scalar_oracle": -7.119679927825928,
   "abs_delta_scalar": 0.006398677825927734,
   "scalar_dtype": "<f2",
   "scalar_bits": "1dc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0514271249994636
  },
  {
   "id": "request-19-q01-option-25",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.0390625,
   "scalar_oracle": -7.052636623382568,
   "abs_delta_scalar": 0.01357412338256836,
   "scalar_dtype": "<f2",
   "scalar_bits": "0ac7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0847678750287741
  },
  {
   "id": "request-19-q01-option-26",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.69140625,
   "scalar_oracle": -7.682580471038818,
   "abs_delta_scalar": 0.00882577896118164,
   "scalar_dtype": "<f2",
   "scalar_bits": "b1c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0567462500184774
  },
  {
   "id": "request-19-q01-option-27",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.49609375,
   "scalar_oracle": -7.514122486114502,
   "abs_delta_scalar": 0.018028736114501953,
   "scalar_dtype": "<f2",
   "scalar_bits": "7fc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0929761670995504
  },
  {
   "id": "request-20-q01-option-00",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -7.78515625,
   "scalar_oracle": -7.80185079574585,
   "abs_delta_scalar": 0.01669454574584961,
   "scalar_dtype": "<f2",
   "scalar_bits": "c9c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0365337500115857
  },
  {
   "id": "request-20-q01-option-01",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -7.390625,
   "scalar_oracle": -7.406404495239258,
   "abs_delta_scalar": 0.015779495239257812,
   "scalar_dtype": "<f2",
   "scalar_bits": "64c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0311821249779314
  },
  {
   "id": "request-20-q01-option-02",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.3125,
   "scalar_oracle": -8.328375816345215,
   "abs_delta_scalar": 0.015875816345214844,
   "scalar_dtype": "<f2",
   "scalar_bits": "28c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0535671670222655
  },
  {
   "id": "request-20-q01-option-03",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.359375,
   "scalar_oracle": -8.3722505569458,
   "abs_delta_scalar": 0.012875556945800781,
   "scalar_dtype": "<f2",
   "scalar_bits": "2ec8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0438543329946697
  },
  {
   "id": "request-20-q01-option-04",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": 2.419921875,
   "scalar_oracle": 2.405498504638672,
   "abs_delta_scalar": 0.014423370361328125,
   "scalar_dtype": "<f2",
   "scalar_bits": "d740",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 3.5543917089235038
  },
  {
   "id": "request-20-q01-option-05",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -5.3359375,
   "scalar_oracle": -5.341571807861328,
   "abs_delta_scalar": 0.005634307861328125,
   "scalar_dtype": "<f2",
   "scalar_bits": "56c5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5382635419955477
  },
  {
   "id": "request-20-q01-option-06",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.9140625,
   "scalar_oracle": -8.924745559692383,
   "abs_delta_scalar": 0.010683059692382812,
   "scalar_dtype": "<f2",
   "scalar_bits": "75c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5346926660276949
  },
  {
   "id": "request-20-q01-option-07",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -6.8828125,
   "scalar_oracle": -6.8783721923828125,
   "abs_delta_scalar": 0.0044403076171875,
   "scalar_dtype": "<f2",
   "scalar_bits": "e2c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5095326250884682
  },
  {
   "id": "request-20-q01-option-08",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.859375,
   "scalar_oracle": -8.876729011535645,
   "abs_delta_scalar": 0.01735401153564453,
   "scalar_dtype": "<f2",
   "scalar_bits": "6ec8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5197352919494733
  },
  {
   "id": "request-20-q01-option-09",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.1484375,
   "scalar_oracle": -8.154253005981445,
   "abs_delta_scalar": 0.0058155059814453125,
   "scalar_dtype": "<f2",
   "scalar_bits": "13c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5191010000417009
  },
  {
   "id": "request-20-q01-option-10",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -9.203125,
   "scalar_oracle": -9.206731796264648,
   "abs_delta_scalar": 0.0036067962646484375,
   "scalar_dtype": "<f2",
   "scalar_bits": "9ac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.49855012504849583
  },
  {
   "id": "request-20-q01-option-11",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -6.06640625,
   "scalar_oracle": -6.098209381103516,
   "abs_delta_scalar": 0.031803131103515625,
   "scalar_dtype": "<f2",
   "scalar_bits": "11c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.520011292072013
  },
  {
   "id": "request-20-q01-option-12",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -1.203125,
   "scalar_oracle": -1.189605712890625,
   "abs_delta_scalar": 0.013519287109375,
   "scalar_dtype": "<f2",
   "scalar_bits": "d0bc",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.4984981659799814
  },
  {
   "id": "request-20-q01-option-13",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -9.8359375,
   "scalar_oracle": -9.841769218444824,
   "abs_delta_scalar": 0.005831718444824219,
   "scalar_dtype": "<f2",
   "scalar_bits": "ebc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5211206249659881
  },
  {
   "id": "request-20-q01-option-14",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -10.5,
   "scalar_oracle": -10.507135391235352,
   "abs_delta_scalar": 0.0071353912353515625,
   "scalar_dtype": "<f2",
   "scalar_bits": "40c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5199225000105798
  },
  {
   "id": "request-20-q01-option-15",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -10.265625,
   "scalar_oracle": -10.270621299743652,
   "abs_delta_scalar": 0.004996299743652344,
   "scalar_dtype": "<f2",
   "scalar_bits": "22c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.521651207935065
  },
  {
   "id": "request-20-q01-option-16",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.25,
   "scalar_oracle": -6.233147144317627,
   "abs_delta_scalar": 0.016852855682373047,
   "scalar_dtype": "<f2",
   "scalar_bits": "40c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.541227790992707
  },
  {
   "id": "request-20-q01-option-17",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -10.7421875,
   "scalar_oracle": -10.75239372253418,
   "abs_delta_scalar": 0.010206222534179688,
   "scalar_dtype": "<f2",
   "scalar_bits": "5fc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5241822499083355
  },
  {
   "id": "request-20-q01-option-18",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.1015625,
   "scalar_oracle": -8.108002662658691,
   "abs_delta_scalar": 0.006440162658691406,
   "scalar_dtype": "<f2",
   "scalar_bits": "0dc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.8617816250771284
  },
  {
   "id": "request-20-q01-option-19",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -7.09375,
   "scalar_oracle": -7.108526229858398,
   "abs_delta_scalar": 0.014776229858398438,
   "scalar_dtype": "<f2",
   "scalar_bits": "18c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.49741058400832117
  },
  {
   "id": "request-20-q01-option-20",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -2.53515625,
   "scalar_oracle": -2.5550146102905273,
   "abs_delta_scalar": 0.019858360290527344,
   "scalar_dtype": "<f2",
   "scalar_bits": "12c1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5164714589482173
  },
  {
   "id": "request-20-q01-option-21",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -7.37890625,
   "scalar_oracle": -7.36898946762085,
   "abs_delta_scalar": 0.00991678237915039,
   "scalar_dtype": "<f2",
   "scalar_bits": "61c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.51677087508142
  },
  {
   "id": "request-20-q01-option-22",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -6.9140625,
   "scalar_oracle": -6.927596569061279,
   "abs_delta_scalar": 0.013534069061279297,
   "scalar_dtype": "<f2",
   "scalar_bits": "eac6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5157493751030415
  },
  {
   "id": "request-20-q01-option-23",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -8.125,
   "scalar_oracle": -8.134787559509277,
   "abs_delta_scalar": 0.009787559509277344,
   "scalar_dtype": "<f2",
   "scalar_bits": "10c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5144704999402165
  },
  {
   "id": "request-20-q01-option-24",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -7.984375,
   "scalar_oracle": -8.004584312438965,
   "abs_delta_scalar": 0.020209312438964844,
   "scalar_dtype": "<f2",
   "scalar_bits": "fcc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5029552499763668
  },
  {
   "id": "request-20-q01-option-25",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -4.9453125,
   "scalar_oracle": -4.9726715087890625,
   "abs_delta_scalar": 0.0273590087890625,
   "scalar_dtype": "<f2",
   "scalar_bits": "f2c4",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5287950409110636
  },
  {
   "id": "request-20-q01-option-26",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -5.60546875,
   "scalar_oracle": -5.6035261154174805,
   "abs_delta_scalar": 0.0019426345825195312,
   "scalar_dtype": "<f2",
   "scalar_bits": "9bc5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5225455830805004
  },
  {
   "id": "request-20-q01-option-27",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -3.224609375,
   "scalar_oracle": -3.2340099811553955,
   "abs_delta_scalar": 0.009400606155395508,
   "scalar_dtype": "<f2",
   "scalar_bits": "73c2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5253694169223309
  },
  {
   "id": "request-20-q01-option-28",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.1953125,
   "scalar_oracle": -8.21119499206543,
   "abs_delta_scalar": 0.015882492065429688,
   "scalar_dtype": "<f2",
   "scalar_bits": "19c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5240156670333818
  },
  {
   "id": "request-20-q01-option-29",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -9.203125,
   "scalar_oracle": -9.236793518066406,
   "abs_delta_scalar": 0.03366851806640625,
   "scalar_dtype": "<f2",
   "scalar_bits": "9ac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5244761250214651
  },
  {
   "id": "request-20-q01-option-30",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -7.89453125,
   "scalar_oracle": -7.896141052246094,
   "abs_delta_scalar": 0.00160980224609375,
   "scalar_dtype": "<f2",
   "scalar_bits": "e5c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5031404999317601
  },
  {
   "id": "request-20-q01-option-31",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": 4.1953125,
   "scalar_oracle": 4.17133903503418,
   "abs_delta_scalar": 0.023973464965820312,
   "scalar_dtype": "<f2",
   "scalar_bits": "3244",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5202645410317928
  }
 ],
 "questions": [
  {
   "id": "request-01-q01",
   "request_id": "request-01",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-01-q01-option-00",
    "request-01-q01-option-01"
   ],
   "options": [
    "electrical repair",
    "surface cleaning"
   ],
   "nopts": 2,
   "scalars": [
    4.3125,
    -8.171875
   ],
   "scalars_oracle": [
    4.314200401306152,
    -8.20827865600586
   ],
   "p": [
    0.9992030557515728,
    0.0007969442484272865
   ],
   "p_oracle": [
    0.9992202072785928,
    0.0007797927214071797
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "electrical repair",
   "oracle_selected_option": "electrical repair",
   "oracle_margin": 0.9984404145571857,
   "abs_delta_p": [
    1.7151527020087265e-05,
    1.715152702010678e-05
   ],
   "max_abs_delta_p": 1.715152702010678e-05,
   "mean_abs_delta_p": 1.7151527020097023e-05,
   "finite": true
  },
  {
   "id": "request-02-q01",
   "request_id": "request-02",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-02-q01-option-00",
    "request-02-q01-option-01"
   ],
   "options": [
    "late delivery",
    "wrong color"
   ],
   "nopts": 2,
   "scalars": [
    6.23828125,
    -9.421875
   ],
   "scalars_oracle": [
    6.207533836364746,
    -9.434723854064941
   ],
   "p": [
    0.9998701058431072,
    0.00012989415689273708
   ],
   "p_oracle": [
    0.9998687706764677,
    0.00013122932353227463
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "late delivery",
   "oracle_selected_option": "late delivery",
   "oracle_margin": 0.9997375413529355,
   "abs_delta_p": [
    1.335166639471197e-06,
    1.3351666395375501e-06
   ],
   "max_abs_delta_p": 1.3351666395375501e-06,
   "mean_abs_delta_p": 1.3351666395043735e-06,
   "finite": true
  },
  {
   "id": "request-03-q01",
   "request_id": "request-03",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-03-q01-option-00",
    "request-03-q01-option-01"
   ],
   "options": [
    "a damaged hinge",
    "a missing label"
   ],
   "nopts": 2,
   "scalars": [
    6.625,
    -8.2109375
   ],
   "scalars_oracle": [
    6.595953941345215,
    -8.22585678100586
   ],
   "p": [
    0.9997919821388277,
    0.00020801786117240992
   ],
   "p_oracle": [
    0.9997902964838274,
    0.00020970351617246997
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "a damaged hinge",
   "oracle_selected_option": "a damaged hinge",
   "oracle_margin": 0.999580592967655,
   "abs_delta_p": [
    1.6856550002497528e-06,
    1.6856550000600445e-06
   ],
   "max_abs_delta_p": 1.6856550002497528e-06,
   "mean_abs_delta_p": 1.6856550001548987e-06,
   "finite": true
  },
  {
   "id": "request-04-q01",
   "request_id": "request-04",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-04-q01-option-00",
    "request-04-q01-option-01"
   ],
   "options": [
    "the cooling equipment",
    "the floor covering"
   ],
   "nopts": 2,
   "scalars": [
    5.984375,
    -10.546875
   ],
   "scalars_oracle": [
    5.953964710235596,
    -10.532769203186035
   ],
   "p": [
    0.9999210351340642,
    7.89648659356947e-05
   ],
   "p_oracle": [
    0.9999190008429476,
    8.099915705234445e-05
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "the cooling equipment",
   "oracle_selected_option": "the cooling equipment",
   "oracle_margin": 0.9998380016858952,
   "abs_delta_p": [
    2.0342911165593947e-06,
    2.0342911166497494e-06
   ],
   "max_abs_delta_p": 2.0342911166497494e-06,
   "mean_abs_delta_p": 2.034291116604572e-06,
   "finite": true
  },
  {
   "id": "request-05-q01",
   "request_id": "request-05",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-05-q01-option-00",
    "request-05-q01-option-01"
   ],
   "options": [
    "a duplicate charge",
    "a delivery delay"
   ],
   "nopts": 2,
   "scalars": [
    7.65234375,
    -8.84375
   ],
   "scalars_oracle": [
    7.658699989318848,
    -8.862298965454102
   ],
   "p": [
    0.9999194328738734,
    8.056712612653072e-05
   ],
   "p_oracle": [
    0.9999205712578337,
    7.942874216629776e-05
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "a duplicate charge",
   "oracle_selected_option": "a duplicate charge",
   "oracle_margin": 0.9998411425156675,
   "abs_delta_p": [
    1.1383839603196222e-06,
    1.1383839602329538e-06
   ],
   "max_abs_delta_p": 1.1383839603196222e-06,
   "mean_abs_delta_p": 1.138383960276288e-06,
   "finite": true
  },
  {
   "id": "request-06-q01",
   "request_id": "request-06",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-06-q01-option-00",
    "request-06-q01-option-01"
   ],
   "options": [
    "stop the water",
    "repaint the ceiling"
   ],
   "nopts": 2,
   "scalars": [
    -0.779296875,
    -8.265625
   ],
   "scalars_oracle": [
    -0.843937337398529,
    -8.322973251342773
   ],
   "p": [
    0.9863180548999116,
    0.013681945100088327
   ],
   "p_oracle": [
    0.9862617084759727,
    0.0137382915240273
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "stop the water",
   "oracle_selected_option": "stop the water",
   "oracle_margin": 0.9725234169519454,
   "abs_delta_p": [
    5.634642393892442e-05,
    5.634642393897299e-05
   ],
   "max_abs_delta_p": 5.634642393897299e-05,
   "mean_abs_delta_p": 5.63464239389487e-05,
   "finite": true
  },
  {
   "id": "request-07-q01",
   "request_id": "request-07",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-07-q01-option-00",
    "request-07-q01-option-01",
    "request-07-q01-option-02",
    "request-07-q01-option-03",
    "request-07-q01-option-04",
    "request-07-q01-option-05",
    "request-07-q01-option-06",
    "request-07-q01-option-07",
    "request-07-q01-option-08"
   ],
   "options": [
    "plants",
    "rocks",
    "stars",
    "poems",
    "roads",
    "music",
    "clothing",
    "numbers",
    "weather"
   ],
   "nopts": 9,
   "scalars": [
    4.14453125,
    -9.671875,
    -9.296875,
    -5.69140625,
    -10.5859375,
    -8.9609375,
    -9.8046875,
    -6.8125,
    -6.78125
   ],
   "scalars_oracle": [
    4.116079330444336,
    -9.69235897064209,
    -9.30652904510498,
    -5.676880836486816,
    -10.599185943603516,
    -8.972684860229492,
    -9.825861930847168,
    -6.823195457458496,
    -6.788428783416748
   ],
   "p": [
    0.9906532429582852,
    0.00036908632514739395,
    0.0004572893193301694,
    0.0035888393615696036,
    0.00021892060538507293,
    0.0005540641975948948,
    0.00034211184064253765,
    0.001891185446110202,
    0.0019252599459350455
   ],
   "p_oracle": [
    0.9905073080192877,
    0.0003707160297848232,
    0.00046215974250602646,
    0.0036775251265417186,
    0.00022079827667603936,
    0.0005592959160818941,
    0.00034348689011294163,
    0.0019101906994275666,
    0.0019485192995812336
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "plants",
   "oracle_selected_option": "plants",
   "oracle_margin": 0.9868297828927459,
   "abs_delta_p": [
    0.0001459349389975051,
    1.6297046374292505e-06,
    4.870423175857067e-06,
    8.868576497211497e-05,
    1.877671290966427e-06,
    5.231718486999244e-06,
    1.375049470403976e-06,
    1.9005253317364587e-05,
    2.3259353646188155e-05
   ],
   "max_abs_delta_p": 0.0001459349389975051,
   "mean_abs_delta_p": 3.2429986443869863e-05,
   "finite": true
  },
  {
   "id": "request-07-q02",
   "request_id": "request-07",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-07-q02-option-00",
    "request-07-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    3.23046875,
    -1.458984375
   ],
   "scalars_oracle": [
    3.2154455184936523,
    -1.4813950061798096
   ],
   "p": [
    0.9358173563909802,
    0.06418264360901972
   ],
   "p_oracle": [
    0.9360704396058132,
    0.06392956039418676
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.8721408792116264,
   "abs_delta_p": [
    0.000253083214832972,
    0.00025308321483295815
   ],
   "max_abs_delta_p": 0.000253083214832972,
   "mean_abs_delta_p": 0.0002530832148329651,
   "finite": true
  },
  {
   "id": "request-07-q03",
   "request_id": "request-07",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-07-q03-option-00",
    "request-07-q03-option-01",
    "request-07-q03-option-02",
    "request-07-q03-option-03",
    "request-07-q03-option-04",
    "request-07-q03-option-05",
    "request-07-q03-option-06",
    "request-07-q03-option-07",
    "request-07-q03-option-08",
    "request-07-q03-option-09"
   ],
   "options": [
    "level 1",
    "level 2",
    "level 3",
    "level 4",
    "level 5",
    "level 6",
    "level 7",
    "level 8",
    "level 9",
    "level 10"
   ],
   "nopts": 10,
   "scalars": [
    -0.18896484375,
    1.06640625,
    0.98291015625,
    0.75390625,
    0.443115234375,
    0.7021484375,
    0.765625,
    0.6796875,
    -0.2161865234375,
    -1.689453125
   ],
   "scalars_oracle": [
    -0.1996888518333435,
    1.066279649734497,
    0.9806243181228638,
    0.7489750385284424,
    0.4373612403869629,
    0.6929409503936768,
    0.7530665397644043,
    0.6613963842391968,
    -0.23028063774108887,
    -1.719403862953186
   ],
   "p": [
    0.06852844678950806,
    0.14041521453796316,
    0.13387302790675262,
    0.11745231516189299,
    0.09834065836381406,
    0.11402942509386077,
    0.11824146549641584,
    0.11257522991562542,
    0.06747071791742688,
    0.029073498816740265
   ],
   "p_oracle": [
    0.06844557046678758,
    0.1410972557943501,
    0.13435741245143595,
    0.11769923201661937,
    0.09850107499890995,
    0.11399026048335885,
    0.11797473484160344,
    0.11195394081789756,
    0.0672594693782866,
    0.028721048750750485
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "level 2",
   "oracle_selected_option": "level 2",
   "oracle_margin": 0.006739843342914137,
   "abs_delta_p": [
    8.287632272048484e-05,
    0.0006820412563869338,
    0.0004843845446833306,
    0.0002469168547263867,
    0.0001604166350958891,
    3.9164610501921104e-05,
    0.00026673065481240543,
    0.0006212890977278646,
    0.00021124853914028519,
    0.0003524500659897803
   ],
   "max_abs_delta_p": 0.0006820412563869338,
   "mean_abs_delta_p": 0.0003147518581785282,
   "finite": true
  },
  {
   "id": "request-08-q01",
   "request_id": "request-08",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-08-q01-option-00",
    "request-08-q01-option-01",
    "request-08-q01-option-02",
    "request-08-q01-option-03",
    "request-08-q01-option-04",
    "request-08-q01-option-05",
    "request-08-q01-option-06",
    "request-08-q01-option-07",
    "request-08-q01-option-08",
    "request-08-q01-option-09"
   ],
   "options": [
    "lock",
    "roof",
    "window",
    "carpet",
    "wall",
    "drain",
    "lamp",
    "shelf",
    "chair",
    "curtain"
   ],
   "nopts": 10,
   "scalars": [
    0.93359375,
    -9.0546875,
    -7.87890625,
    -11.46875,
    -7.28515625,
    -9.6640625,
    -9.515625,
    -10.34375,
    -9.578125,
    -10.6953125
   ],
   "scalars_oracle": [
    0.9249671101570129,
    -9.055648803710938,
    -7.874567985534668,
    -11.47532844543457,
    -7.288543701171875,
    -9.6605863571167,
    -9.511998176574707,
    -10.34797191619873,
    -9.580318450927734,
    -10.699100494384766
   ],
   "p": [
    0.9708415792415696,
    0.003223842648300393,
    0.006311977488615954,
    0.0008114842760479953,
    0.008861665542053856,
    0.002275863933672558,
    0.002477329486306666,
    0.0015433677420367813,
    0.0023904147208011154,
    0.0012624749205950667
   ],
   "p_oracle": [
    0.9707076233156018,
    0.0032375478941282032,
    0.0063580361304097265,
    0.0008123224916065292,
    0.00888700915380173,
    0.002291341877189737,
    0.0024943923445487387,
    0.0015470437789951472,
    0.0023988872727527663,
    0.00126579574096568
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "lock",
   "oracle_selected_option": "lock",
   "oracle_margin": 0.9618206141618001,
   "abs_delta_p": [
    0.00013395592596776584,
    1.3705245827810072e-05,
    4.6058641793772503e-05,
    8.382155585338804e-07,
    2.5343611747873432e-05,
    1.547794351717917e-05,
    1.7062858242072876e-05,
    3.6760369583658213e-06,
    8.472551951650871e-06,
    3.320820370613254e-06
   ],
   "max_abs_delta_p": 0.00013395592596776584,
   "mean_abs_delta_p": 2.6791185193563773e-05,
   "finite": true
  },
  {
   "id": "request-08-q02",
   "request_id": "request-08",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-08-q02-option-00",
    "request-08-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    3.578125,
    -1.326171875
   ],
   "scalars_oracle": [
    3.5709025859832764,
    -1.3289239406585693
   ],
   "p": [
    0.9428083628456251,
    0.057191637154374975
   ],
   "p_oracle": [
    0.942670467195647,
    0.05732953280435297
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.885340934391294,
   "abs_delta_p": [
    0.00013789564997812587,
    0.00013789564997799403
   ],
   "max_abs_delta_p": 0.00013789564997812587,
   "mean_abs_delta_p": 0.00013789564997805995,
   "finite": true
  },
  {
   "id": "request-08-q03",
   "request_id": "request-08",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-08-q03-option-00",
    "request-08-q03-option-01",
    "request-08-q03-option-02",
    "request-08-q03-option-03",
    "request-08-q03-option-04",
    "request-08-q03-option-05",
    "request-08-q03-option-06",
    "request-08-q03-option-07",
    "request-08-q03-option-08",
    "request-08-q03-option-09"
   ],
   "options": [
    "level 1",
    "level 2",
    "level 3",
    "level 4",
    "level 5",
    "level 6",
    "level 7",
    "level 8",
    "level 9",
    "level 10"
   ],
   "nopts": 10,
   "scalars": [
    -3.30859375,
    -1.8974609375,
    -1.4248046875,
    -1.3857421875,
    -1.482421875,
    -1.203125,
    -1.2607421875,
    -1.3466796875,
    -1.4814453125,
    -2.38671875
   ],
   "scalars_oracle": [
    -3.3124825954437256,
    -1.9047918319702148,
    -1.4303022623062134,
    -1.3922051191329956,
    -1.48871910572052,
    -1.2124919891357422,
    -1.2709437608718872,
    -1.354970097541809,
    -1.4879703521728516,
    -2.396772861480713
   ],
   "p": [
    0.03821268801456236,
    0.08558664157346375,
    0.1121254686745306,
    0.11465641135757645,
    0.10849394805786149,
    0.1272676828719541,
    0.12314573609528758,
    0.1172444835308317,
    0.10855450845152134,
    0.06471243137241071
   ],
   "p_oracle": [
    0.03829370334805408,
    0.08559956498729611,
    0.11225994259558048,
    0.11473061408817004,
    0.10857444261658075,
    0.12713889005910925,
    0.12296245919395053,
    0.11719791061365697,
    0.10862090712691694,
    0.06462156537068481
   ],
   "argmax": 5,
   "oracle_argmax": 5,
   "argmax_agrees": true,
   "selected_option": "level 6",
   "oracle_selected_option": "level 6",
   "oracle_margin": 0.004176430865158717,
   "abs_delta_p": [
    8.101533349171713e-05,
    1.292341383235529e-05,
    0.00013447392104987133,
    7.420273059358107e-05,
    8.04945587192657e-05,
    0.00012879281284486344,
    0.00018327690133704277,
    4.6572917174719786e-05,
    6.639867539559585e-05,
    9.086600172590609e-05
   ],
   "max_abs_delta_p": 0.00018327690133704277,
   "mean_abs_delta_p": 8.990172661649185e-05,
   "finite": true
  },
  {
   "id": "request-09-q01",
   "request_id": "request-09",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-09-q01-option-00",
    "request-09-q01-option-01",
    "request-09-q01-option-02",
    "request-09-q01-option-03",
    "request-09-q01-option-04",
    "request-09-q01-option-05",
    "request-09-q01-option-06",
    "request-09-q01-option-07",
    "request-09-q01-option-08",
    "request-09-q01-option-09",
    "request-09-q01-option-10"
   ],
   "options": [
    "rain",
    "snow",
    "fog",
    "heat",
    "frost",
    "wind",
    "dust",
    "smoke",
    "sunshine",
    "hail",
    "drought"
   ],
   "nopts": 11,
   "scalars": [
    0.79541015625,
    -7.54296875,
    -8.2890625,
    -8.6171875,
    -8.46875,
    -6.92578125,
    -9.328125,
    -10.5,
    -10.59375,
    -7.95703125,
    -8.578125
   ],
   "scalars_oracle": [
    0.7694981694221497,
    -7.545819282531738,
    -8.288448333740234,
    -8.621004104614258,
    -8.477935791015625,
    -6.928310871124268,
    -9.343143463134766,
    -10.518312454223633,
    -10.605207443237305,
    -7.95442008972168,
    -8.586217880249023
   ],
   "p": [
    0.9492691449971163,
    0.008092230639249885,
    0.005283375196844513,
    0.0043800718804562055,
    0.004767807539393093,
    0.011514222293801647,
    0.0029177568975879125,
    0.0014935745566339571,
    0.001415667072877333,
    0.006387208242552439,
    0.004478940683486447
   ],
   "p_oracle": [
    0.9486699494562557,
    0.008194399866555504,
    0.005360683684473484,
    0.00443292506160241,
    0.00481055745786062,
    0.011661734510957271,
    0.0029341229642238504,
    0.0014991277682935508,
    0.0014265075320346937,
    0.006488068030313924,
    0.00452192366742885
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "rain",
   "oracle_selected_option": "rain",
   "oracle_margin": 0.9370082149452984,
   "abs_delta_p": [
    0.0005991955408606309,
    0.00010216922730561968,
    7.730848762897095e-05,
    5.2853181146204214e-05,
    4.274991846752677e-05,
    0.00014751221715562403,
    1.636606663593795e-05,
    5.553211659593653e-06,
    1.0840459157360595e-05,
    0.00010085978776148453,
    4.298298394240282e-05
   ],
   "max_abs_delta_p": 0.0005991955408606309,
   "mean_abs_delta_p": 0.00010894464379285055,
   "finite": true
  },
  {
   "id": "request-09-q02",
   "request_id": "request-09",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-09-q02-option-00",
    "request-09-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    -4.53125,
    2.421875
   ],
   "scalars_oracle": [
    -4.542425155639648,
    2.413087844848633
   ],
   "p": [
    0.018465477371010522,
    0.9815345226289894
   ],
   "p_oracle": [
    0.018440761430690376,
    0.9815592385693096
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "no",
   "oracle_selected_option": "no",
   "oracle_margin": 0.9631184771386192,
   "abs_delta_p": [
    2.4715940320146207e-05,
    2.4715940320163554e-05
   ],
   "max_abs_delta_p": 2.4715940320163554e-05,
   "mean_abs_delta_p": 2.471594032015488e-05,
   "finite": true
  },
  {
   "id": "request-09-q03",
   "request_id": "request-09",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-09-q03-option-00",
    "request-09-q03-option-01"
   ],
   "options": [
    "dry",
    "wet"
   ],
   "nopts": 2,
   "scalars": [
    -6.79296875,
    2.158203125
   ],
   "scalars_oracle": [
    -6.80530309677124,
    2.1344544887542725
   ],
   "p": [
    0.005970386355465352,
    0.9940296136445348
   ],
   "p_oracle": [
    0.0060092204092440865,
    0.9939907795907559
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "wet",
   "oracle_selected_option": "wet",
   "oracle_margin": 0.9879815591815118,
   "abs_delta_p": [
    3.883405377873468e-05,
    3.883405377891336e-05
   ],
   "max_abs_delta_p": 3.883405377891336e-05,
   "mean_abs_delta_p": 3.883405377882402e-05,
   "finite": true
  },
  {
   "id": "request-10-q01",
   "request_id": "request-10",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-10-q01-option-00",
    "request-10-q01-option-01",
    "request-10-q01-option-02",
    "request-10-q01-option-03",
    "request-10-q01-option-04",
    "request-10-q01-option-05",
    "request-10-q01-option-06",
    "request-10-q01-option-07",
    "request-10-q01-option-08",
    "request-10-q01-option-09",
    "request-10-q01-option-10",
    "request-10-q01-option-11"
   ],
   "options": [
    "rice",
    "sand",
    "water",
    "stones",
    "paper",
    "coins",
    "soil",
    "paint",
    "thread",
    "salt",
    "flour",
    "leaves"
   ],
   "nopts": 12,
   "scalars": [
    0.86767578125,
    -9.265625,
    -9.3515625,
    -11.4140625,
    -11.2734375,
    -11.75,
    -11.5625,
    -11.3046875,
    -7.44921875,
    -8.75,
    -10.1875,
    -11.390625
   ],
   "scalars_oracle": [
    0.8400804996490479,
    -9.282438278198242,
    -9.375736236572266,
    -11.433075904846191,
    -11.280050277709961,
    -11.762323379516602,
    -11.585577011108398,
    -11.320062637329102,
    -7.437173843383789,
    -8.777253150939941,
    -10.206853866577148,
    -11.40426254272461
   ],
   "p": [
    0.9748546066478319,
    0.002979724209853647,
    0.002836933190916832,
    0.000872975469847413,
    0.0009460208411435163,
    0.0007204983829154316,
    0.0008019818913020616,
    0.0009292775504237859,
    0.008412992000993623,
    0.0040007170451757884,
    0.0017595270210959777,
    0.0008847457485001996
   ],
   "p_oracle": [
    0.9745978148411945,
    0.0029973496368662185,
    0.002841736544796916,
    0.0008770359084199525,
    0.0009571797033756986,
    0.0007266220906150506,
    0.0008038433781029951,
    0.0009355428472892288,
    0.008603467018522446,
    0.004000445217858127,
    0.0017673671702117771,
    0.0008915956427471904
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "rice",
   "oracle_selected_option": "rice",
   "oracle_margin": 0.965994347822672,
   "abs_delta_p": [
    0.000256791806637402,
    1.7625427012571415e-05,
    4.803353880083929e-06,
    4.0604385725394624e-06,
    1.115886223218229e-05,
    6.123707699618982e-06,
    1.8614868009335763e-06,
    6.265296865442908e-06,
    0.00019047501752882286,
    2.718273176610317e-07,
    7.840149115799413e-06,
    6.849894246990871e-06
   ],
   "max_abs_delta_p": 0.000256791806637402,
   "mean_abs_delta_p": 4.284393899250406e-05,
   "finite": true
  },
  {
   "id": "request-10-q02",
   "request_id": "request-10",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-10-q02-option-00",
    "request-10-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    2.751953125,
    -2.48828125
   ],
   "scalars_oracle": [
    2.7351107597351074,
    -2.4982452392578125
   ],
   "p": [
    0.9523213872823875,
    0.04767861271761263
   ],
   "p_oracle": [
    0.9521426038586818,
    0.04785739614131819
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.9042852077173635,
   "abs_delta_p": [
    0.00017878342370569733,
    0.00017878342370555855
   ],
   "max_abs_delta_p": 0.00017878342370569733,
   "mean_abs_delta_p": 0.00017878342370562794,
   "finite": true
  },
  {
   "id": "request-10-q03",
   "request_id": "request-10",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-10-q03-option-00",
    "request-10-q03-option-01"
   ],
   "options": [
    "empty",
    "nonempty"
   ],
   "nopts": 2,
   "scalars": [
    -2.599609375,
    2.80859375
   ],
   "scalars_oracle": [
    -2.61518931388855,
    2.7923030853271484
   ],
   "p": [
    0.04350491268674578,
    0.9564950873132542
   ],
   "p_oracle": [
    0.04352181575658243,
    0.9564781842434176
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "nonempty",
   "oracle_selected_option": "nonempty",
   "oracle_margin": 0.9129563684868351,
   "abs_delta_p": [
    1.690306983664902e-05,
    1.690306983659351e-05
   ],
   "max_abs_delta_p": 1.690306983664902e-05,
   "mean_abs_delta_p": 1.6903069836621265e-05,
   "finite": true
  },
  {
   "id": "request-11-q01",
   "request_id": "request-11",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-11-q01-option-00",
    "request-11-q01-option-01",
    "request-11-q01-option-02",
    "request-11-q01-option-03",
    "request-11-q01-option-04",
    "request-11-q01-option-05",
    "request-11-q01-option-06",
    "request-11-q01-option-07",
    "request-11-q01-option-08",
    "request-11-q01-option-09",
    "request-11-q01-option-10",
    "request-11-q01-option-11",
    "request-11-q01-option-12",
    "request-11-q01-option-13"
   ],
   "options": [
    "cotton",
    "wool",
    "silk",
    "linen",
    "wood",
    "glass",
    "stone",
    "steel",
    "copper",
    "rubber",
    "paper",
    "clay",
    "leather",
    "plastic"
   ],
   "nopts": 14,
   "scalars": [
    5.2578125,
    -6.21875,
    -7.16015625,
    -4.40234375,
    -6.59375,
    -7.44140625,
    -9.1953125,
    -8.3984375,
    -8.6171875,
    -9.53125,
    -7.00390625,
    -9.234375,
    -9.3125,
    -8.3515625
   ],
   "scalars_oracle": [
    5.246809482574463,
    -6.228903293609619,
    -7.158008575439453,
    -4.403936386108398,
    -6.593619346618652,
    -7.444218635559082,
    -9.191930770874023,
    -8.394148826599121,
    -8.630069732666016,
    -9.541899681091309,
    -7.013782501220703,
    -9.234328269958496,
    -9.323783874511719,
    -8.359192848205566
   ],
   "p": [
    0.988958462094127,
    0.0014030053530582605,
    0.0008192796404948352,
    0.003961263520159848,
    0.001132390519596883,
    0.0006976455608032393,
    0.0002560772196848344,
    0.00040376627416202106,
    0.00035632248631610615,
    0.00021134983634380015,
    0.0008957946319107325,
    0.00025042453303799006,
    0.00023949074010890404,
    0.00041472759019545017
   ],
   "p_oracle": [
    0.9889120205917192,
    0.001403620839647622,
    0.0008254206927579425,
    0.0039824350139213245,
    0.0011395643482980929,
    0.0007008855319549572,
    0.0002581786875110537,
    0.00040729076030881746,
    0.00035592334436555984,
    0.00021138258655764423,
    0.0008963294960782709,
    0.00025199891705764957,
    0.00023944106269725907,
    0.00041550812712453766
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "cotton",
   "oracle_selected_option": "cotton",
   "oracle_margin": 0.9849295855777979,
   "abs_delta_p": [
    4.64415024077347e-05,
    6.154865893615025e-07,
    6.141052263107234e-06,
    2.1171493761476534e-05,
    7.173828701209943e-06,
    3.2399711517178955e-06,
    2.101467826219291e-06,
    3.524486146796405e-06,
    3.991419505463092e-07,
    3.2750213844079824e-08,
    5.348641675383689e-07,
    1.5743840196595052e-06,
    4.967741164496565e-08,
    7.805369290874841e-07
   ],
   "max_abs_delta_p": 4.64415024077347e-05,
   "mean_abs_delta_p": 6.698617395710302e-06,
   "finite": true
  },
  {
   "id": "request-11-q02",
   "request_id": "request-11",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-11-q02-option-00",
    "request-11-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    -3.03515625,
    3.0625
   ],
   "scalars_oracle": [
    -3.04154896736145,
    3.0489227771759033
   ],
   "p": [
    0.029760093945964218,
    0.9702399060540357
   ],
   "p_oracle": [
    0.029878865074433554,
    0.9701211349255664
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "no",
   "oracle_selected_option": "no",
   "oracle_margin": 0.9402422698511328,
   "abs_delta_p": [
    0.00011877112846933602,
    0.00011877112846925275
   ],
   "max_abs_delta_p": 0.00011877112846933602,
   "mean_abs_delta_p": 0.00011877112846929438,
   "finite": true
  },
  {
   "id": "request-11-q03",
   "request_id": "request-11",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-11-q03-option-00",
    "request-11-q03-option-01",
    "request-11-q03-option-02"
   ],
   "options": [
    "none",
    "some",
    "complete"
   ],
   "nopts": 3,
   "scalars": [
    -1.9833984375,
    -0.7919921875,
    0.1658935546875
   ],
   "scalars_oracle": [
    -1.9961133003234863,
    -0.7945132851600647,
    0.13405287265777588
   ],
   "p": [
    0.1564840369177385,
    0.30912866310323367,
    0.5343872999790279
   ],
   "p_oracle": [
    0.15711289848380908,
    0.3121841461234784,
    0.5307029553927125
   ],
   "argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "selected_option": "complete",
   "oracle_selected_option": "complete",
   "oracle_margin": 0.21851880926923412,
   "abs_delta_p": [
    0.0006288615660705876,
    0.0030554830202447203,
    0.003684344586315391
   ],
   "max_abs_delta_p": 0.003684344586315391,
   "mean_abs_delta_p": 0.002456229724210233,
   "finite": true
  },
  {
   "id": "request-12-q01",
   "request_id": "request-12",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-12-q01-option-00",
    "request-12-q01-option-01",
    "request-12-q01-option-02",
    "request-12-q01-option-03",
    "request-12-q01-option-04",
    "request-12-q01-option-05",
    "request-12-q01-option-06",
    "request-12-q01-option-07",
    "request-12-q01-option-08",
    "request-12-q01-option-09",
    "request-12-q01-option-10",
    "request-12-q01-option-11",
    "request-12-q01-option-12",
    "request-12-q01-option-13",
    "request-12-q01-option-14",
    "request-12-q01-option-15"
   ],
   "options": [
    "sparrow",
    "robin",
    "raven",
    "owl",
    "duck",
    "goose",
    "swan",
    "finch",
    "gull",
    "hawk",
    "eagle",
    "dove",
    "heron",
    "crane",
    "wren",
    "lark"
   ],
   "nopts": 16,
   "scalars": [
    6.2578125,
    -3.328125,
    -6.76171875,
    -7.01171875,
    -7.1640625,
    -7.4375,
    -7.15625,
    -2.65234375,
    -6.6015625,
    -7.66796875,
    -8.25,
    -4.796875,
    -8.078125,
    -7.00390625,
    -2.9453125,
    -3.22265625
   ],
   "scalars_oracle": [
    6.2408976554870605,
    -3.3339803218841553,
    -6.770749568939209,
    -7.012792587280273,
    -7.17970609664917,
    -7.4471611976623535,
    -7.164470195770264,
    -2.6470611095428467,
    -6.595610618591309,
    -7.668771743774414,
    -8.25868034362793,
    -4.801041126251221,
    -8.084135055541992,
    -7.016900062561035,
    -2.95692777633667,
    -3.219796895980835
   ],
   "p": [
    0.9744334840592961,
    0.004072176767158894,
    0.0005724236684163825,
    0.0004962214274440874,
    0.00045485044925933115,
    0.00038905420639765077,
    0.0004568855709244137,
    0.005991479658015927,
    0.0006272826606026296,
    0.0003410476669886736,
    0.0002445532233627161,
    0.0017592577276250919,
    0.0002697909104916299,
    0.0004984416538488631,
    0.005067905869107175,
    0.004325144481060197
   ],
   "p_oracle": [
    0.9742324389470528,
    0.004097147806356085,
    0.0005748897092791492,
    0.0005006303058684603,
    0.0004550870721678767,
    0.00039058955678796583,
    0.00045906646207074066,
    0.006066709336247993,
    0.0006354018368255835,
    0.0003441310981245799,
    0.000245655968598753,
    0.001771755060649297,
    0.00027142129832778033,
    0.000499456640004762,
    0.005082227613647842,
    0.00437339128799038
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "sparrow",
   "oracle_selected_option": "sparrow",
   "oracle_margin": 0.9681657296108048,
   "abs_delta_p": [
    0.00020104511224328103,
    2.4971039197191e-05,
    2.4660408627666342e-06,
    4.408878424372862e-06,
    2.3662290854557485e-07,
    1.535350390315061e-06,
    2.180891146326988e-06,
    7.522967823206557e-05,
    8.11917622295388e-06,
    3.0834311359062973e-06,
    1.1027452360369474e-06,
    1.2497333024205203e-05,
    1.6303878361504254e-06,
    1.014986155898831e-06,
    1.4321744540666398e-05,
    4.824680693018245e-05
   ],
   "max_abs_delta_p": 0.00020104511224328103,
   "mean_abs_delta_p": 2.5130639030429072e-05,
   "finite": true
  },
  {
   "id": "request-12-q02",
   "request_id": "request-12",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-12-q02-option-00",
    "request-12-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    3.052734375,
    -2.224609375
   ],
   "scalars_oracle": [
    3.03344464302063,
    -2.245439291000366
   ],
   "p": [
    0.9532750413739076,
    0.04672495862609242
   ],
   "p_oracle": [
    0.9533142271512312,
    0.04668577284876895
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.9066284543024622,
   "abs_delta_p": [
    3.918577732353601e-05,
    3.918577732346662e-05
   ],
   "max_abs_delta_p": 3.918577732353601e-05,
   "mean_abs_delta_p": 3.9185777323501314e-05,
   "finite": true
  },
  {
   "id": "request-12-q03",
   "request_id": "request-12",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-12-q03-option-00",
    "request-12-q03-option-01",
    "request-12-q03-option-02"
   ],
   "options": [
    "still",
    "slightly active",
    "active"
   ],
   "nopts": 3,
   "scalars": [
    -2.693359375,
    -1.3095703125,
    1.52734375
   ],
   "scalars_oracle": [
    -2.706667423248291,
    -1.3185935020446777,
    1.510910987854004
   ],
   "p": [
    0.0696408946665868,
    0.15355956985591065,
    0.7767995354775025
   ],
   "p_oracle": [
    0.06971125616279088,
    0.15409154856746585,
    0.7761971952697433
   ],
   "argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "selected_option": "active",
   "oracle_selected_option": "active",
   "oracle_margin": 0.6221056467022774,
   "abs_delta_p": [
    7.036149620408272e-05,
    0.0005319787115551922,
    0.0006023402077591777
   ],
   "max_abs_delta_p": 0.0006023402077591777,
   "mean_abs_delta_p": 0.00040156013850615085,
   "finite": true
  },
  {
   "id": "request-13-q01",
   "request_id": "request-13",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-13-q01-option-00",
    "request-13-q01-option-01"
   ],
   "options": [
    "send an intact replacement",
    "cancel all future deliveries"
   ],
   "nopts": 2,
   "scalars": [
    4.42578125,
    -9.5859375
   ],
   "scalars_oracle": [
    4.405121803283691,
    -9.618844985961914
   ],
   "p": [
    0.9996668872700176,
    0.00033311272998246975
   ],
   "p_oracle": [
    0.9996692097777317,
    0.0003307902222684505
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "send an intact replacement",
   "oracle_selected_option": "send an intact replacement",
   "oracle_margin": 0.9993384195554632,
   "abs_delta_p": [
    2.3225077140365258e-06,
    2.3225077140192328e-06
   ],
   "max_abs_delta_p": 2.3225077140365258e-06,
   "mean_abs_delta_p": 2.3225077140278793e-06,
   "finite": true
  },
  {
   "id": "request-13-q02",
   "request_id": "request-13",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-13-q02-option-00",
    "request-13-q02-option-01"
   ],
   "options": [
    "the bowl was damaged",
    "the address was missing"
   ],
   "nopts": 2,
   "scalars": [
    5.39453125,
    -10.21875
   ],
   "scalars_oracle": [
    5.3750176429748535,
    -10.238785743713379
   ],
   "p": [
    0.9998665799890902,
    0.00013342001090985438
   ],
   "p_oracle": [
    0.9998666197855478,
    0.00013338021445225527
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "the bowl was damaged",
   "oracle_selected_option": "the bowl was damaged",
   "oracle_margin": 0.9997332395710955,
   "abs_delta_p": [
    3.979645757379302e-08,
    3.979645759910914e-08
   ],
   "max_abs_delta_p": 3.979645759910914e-08,
   "mean_abs_delta_p": 3.979645758645108e-08,
   "finite": true
  },
  {
   "id": "request-13-q03",
   "request_id": "request-13",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-13-q03-option-00",
    "request-13-q03-option-01"
   ],
   "options": [
    "a complaint about damage",
    "a request for directions"
   ],
   "nopts": 2,
   "scalars": [
    7.06640625,
    -10.7890625
   ],
   "scalars_oracle": [
    7.067084312438965,
    -10.796977996826172
   ],
   "p": [
    0.9999629470302284,
    3.705296977158346e-05
   ],
   "p_oracle": [
    0.9999631285300276,
    3.687146997256911e-05
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "a complaint about damage",
   "oracle_selected_option": "a complaint about damage",
   "oracle_margin": 0.999926257060055,
   "abs_delta_p": [
    1.8149979919002135e-07,
    1.814997990143535e-07
   ],
   "max_abs_delta_p": 1.8149979919002135e-07,
   "mean_abs_delta_p": 1.8149979910218742e-07,
   "finite": true
  },
  {
   "id": "request-13-q04",
   "request_id": "request-13",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-13-q04-option-00",
    "request-13-q04-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    4.60546875,
    -1.0859375
   ],
   "scalars_oracle": [
    4.599853515625,
    -1.0872465372085571
   ],
   "p": [
    0.96275323878024,
    0.03724676121975993
   ],
   "p_oracle": [
    0.9626648994801141,
    0.037335100519885804
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.9253297989602284,
   "abs_delta_p": [
    8.833930012586144e-05,
    8.833930012587532e-05
   ],
   "max_abs_delta_p": 8.833930012587532e-05,
   "mean_abs_delta_p": 8.833930012586838e-05,
   "finite": true
  },
  {
   "id": "request-13-q05",
   "request_id": "request-13",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-13-q05-option-00",
    "request-13-q05-option-01",
    "request-13-q05-option-02",
    "request-13-q05-option-03"
   ],
   "options": [
    "unusable",
    "limited",
    "mostly usable",
    "fully usable"
   ],
   "nopts": 4,
   "scalars": [
    1.6240234375,
    0.2861328125,
    -1.291015625,
    -3.94921875
   ],
   "scalars_oracle": [
    1.6196714639663696,
    0.27900874614715576,
    -1.3120677471160889,
    -3.97291898727417
   ],
   "p": [
    0.5896212147307447,
    0.27450552636424547,
    0.11146868669589143,
    0.024404572209118385
   ],
   "p_oracle": [
    0.5906616660481768,
    0.274554667682575,
    0.11060483754235892,
    0.024178828726889312
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "unusable",
   "oracle_selected_option": "unusable",
   "oracle_margin": 0.3161069983656018,
   "abs_delta_p": [
    0.001040451317432134,
    4.91413183295597e-05,
    0.000863849153532506,
    0.00022574348222907312
   ],
   "max_abs_delta_p": 0.001040451317432134,
   "mean_abs_delta_p": 0.0005447963178808182,
   "finite": true
  },
  {
   "id": "request-14-q01",
   "request_id": "request-14",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-14-q01-option-00",
    "request-14-q01-option-01"
   ],
   "options": [
    "a pleasant quiet stay",
    "a noisy delayed arrival"
   ],
   "nopts": 2,
   "scalars": [
    3.595703125,
    -10.8828125
   ],
   "scalars_oracle": [
    3.573038101196289,
    -10.914566040039062
   ],
   "p": [
    0.9997448586044048,
    0.00025514139559523356
   ],
   "p_oracle": [
    0.999746179895068,
    0.000253820104931945
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "a pleasant quiet stay",
   "oracle_selected_option": "a pleasant quiet stay",
   "oracle_margin": 0.9994923597901361,
   "abs_delta_p": [
    1.3212906632142918e-06,
    1.3212906632885596e-06
   ],
   "max_abs_delta_p": 1.3212906632885596e-06,
   "mean_abs_delta_p": 1.3212906632514257e-06,
   "finite": true
  },
  {
   "id": "request-14-q02",
   "request_id": "request-14",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-14-q02-option-00",
    "request-14-q02-option-01"
   ],
   "options": [
    "clean and ready early",
    "dirty and still locked"
   ],
   "nopts": 2,
   "scalars": [
    -0.386962890625,
    -9.7265625
   ],
   "scalars_oracle": [
    -0.42073899507522583,
    -9.746484756469727
   ],
   "p": [
    0.9952123386438988,
    0.0047876613561012955
   ],
   "p_oracle": [
    0.995174470397323,
    0.004825529602676933
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "clean and ready early",
   "oracle_selected_option": "clean and ready early",
   "oracle_margin": 0.9903489407946461,
   "abs_delta_p": [
    3.786824657581178e-05,
    3.786824657563744e-05
   ],
   "max_abs_delta_p": 3.786824657581178e-05,
   "mean_abs_delta_p": 3.786824657572461e-05,
   "finite": true
  },
  {
   "id": "request-14-q03",
   "request_id": "request-14",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-14-q03-option-00",
    "request-14-q03-option-01"
   ],
   "options": [
    "pleased with the experience",
    "angry about the delay"
   ],
   "nopts": 2,
   "scalars": [
    2.4296875,
    -10.453125
   ],
   "scalars_oracle": [
    2.417165756225586,
    -10.482742309570312
   ],
   "p": [
    0.9993652263745825,
    0.0006347736254175325
   ],
   "p_oracle": [
    0.9993713933420288,
    0.0006286066579710768
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "pleased with the experience",
   "oracle_selected_option": "pleased with the experience",
   "oracle_margin": 0.9987427866840578,
   "abs_delta_p": [
    6.166967446374905e-06,
    6.166967446455678e-06
   ],
   "max_abs_delta_p": 6.166967446455678e-06,
   "mean_abs_delta_p": 6.166967446415291e-06,
   "finite": true
  },
  {
   "id": "request-14-q04",
   "request_id": "request-14",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-14-q04-option-00",
    "request-14-q04-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    -1.5673828125,
    1.375
   ],
   "scalars_oracle": [
    -1.5531220436096191,
    1.3508743047714233
   ],
   "p": [
    0.156915253827473,
    0.843084746172527
   ],
   "p_oracle": [
    0.15983899956016281,
    0.8401610004398371
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "no",
   "oracle_selected_option": "no",
   "oracle_margin": 0.6803220008796743,
   "abs_delta_p": [
    0.002923745732689803,
    0.002923745732689942
   ],
   "max_abs_delta_p": 0.002923745732689942,
   "mean_abs_delta_p": 0.0029237457326898725,
   "finite": true
  },
  {
   "id": "request-14-q05",
   "request_id": "request-14",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-14-q05-option-00",
    "request-14-q05-option-01",
    "request-14-q05-option-02",
    "request-14-q05-option-03"
   ],
   "options": [
    "poor",
    "fair",
    "good",
    "excellent"
   ],
   "nopts": 4,
   "scalars": [
    -7.94921875,
    -3.15234375,
    0.01216888427734375,
    -1.2724609375
   ],
   "scalars_oracle": [
    -7.972799301147461,
    -3.1746554374694824,
    -0.007742464542388916,
    -1.2937160730361938
   ],
   "p": [
    0.006391173498634746,
    0.09908611556362411,
    0.6044278209806334,
    0.29009488995710775
   ],
   "p_oracle": [
    0.00638015986060032,
    0.09898711073805981,
    0.6046526754736612,
    0.28998005392767867
   ],
   "argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "selected_option": "good",
   "oracle_selected_option": "good",
   "oracle_margin": 0.3146726215459825,
   "abs_delta_p": [
    1.1013638034426329e-05,
    9.900482556429724e-05,
    0.00022485449302778182,
    0.00011483602942907734
   ],
   "max_abs_delta_p": 0.00022485449302778182,
   "mean_abs_delta_p": 0.00011242724651389568,
   "finite": true
  },
  {
   "id": "request-15-q01",
   "request_id": "request-15",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-15-q01-option-00",
    "request-15-q01-option-01"
   ],
   "options": [
    "arrange an urgent repair",
    "update the wall color"
   ],
   "nopts": 2,
   "scalars": [
    -0.650390625,
    -14.578125
   ],
   "scalars_oracle": [
    -0.6910357475280762,
    -14.593600273132324
   ],
   "p": [
    0.9996505167424641,
    0.00034948325753590416
   ],
   "p_oracle": [
    0.9996454556780844,
    0.00035454432191554705
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "arrange an urgent repair",
   "oracle_selected_option": "arrange an urgent repair",
   "oracle_margin": 0.9992909113561689,
   "abs_delta_p": [
    5.061064379696667e-06,
    5.0610643796428906e-06
   ],
   "max_abs_delta_p": 5.061064379696667e-06,
   "mean_abs_delta_p": 5.061064379669779e-06,
   "finite": true
  },
  {
   "id": "request-15-q02",
   "request_id": "request-15",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-15-q02-option-00",
    "request-15-q02-option-01"
   ],
   "options": [
    "an alternative access route",
    "a printed lunch menu"
   ],
   "nopts": 2,
   "scalars": [
    6.671875,
    -14.5625
   ],
   "scalars_oracle": [
    6.661491870880127,
    -14.577804565429688
   ],
   "p": [
    0.9999946259779542,
    5.374022045837221e-06
   ],
   "p_oracle": [
    0.9999946410697321,
    5.358930267851325e-06
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "an alternative access route",
   "oracle_selected_option": "an alternative access route",
   "oracle_margin": 0.9999892821394643,
   "abs_delta_p": [
    1.5091777938458506e-08,
    1.509177798589574e-08
   ],
   "max_abs_delta_p": 1.509177798589574e-08,
   "mean_abs_delta_p": 1.5091777962177123e-08,
   "finite": true
  },
  {
   "id": "request-15-q03",
   "request_id": "request-15",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-15-q03-option-00",
    "request-15-q03-option-01"
   ],
   "options": [
    "building access and maintenance",
    "gardening plans and supplies"
   ],
   "nopts": 2,
   "scalars": [
    -2.009765625,
    -10.9921875
   ],
   "scalars_oracle": [
    -2.031851053237915,
    -11.027084350585938
   ],
   "p": [
    0.9941346616392029,
    0.005865338360797101
   ],
   "p_oracle": [
    0.99417719477895,
    0.005822805221049984
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "building access and maintenance",
   "oracle_selected_option": "building access and maintenance",
   "oracle_margin": 0.9883543895579,
   "abs_delta_p": [
    4.253313974700301e-05,
    4.253313974711664e-05
   ],
   "max_abs_delta_p": 4.253313974711664e-05,
   "mean_abs_delta_p": 4.2533139747059825e-05,
   "finite": true
  },
  {
   "id": "request-15-q04",
   "request_id": "request-15",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-15-q04-option-00",
    "request-15-q04-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    -1.71875,
    1.373046875
   ],
   "scalars_oracle": [
    -1.7337946891784668,
    1.3656667470932007
   ],
   "p": [
    0.1459480768515914,
    0.8540519231484086
   ],
   "p_oracle": [
    0.14540299930898767,
    0.8545970006910122
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "no",
   "oracle_selected_option": "no",
   "oracle_margin": 0.7091940013820245,
   "abs_delta_p": [
    0.0005450775426037169,
    0.0005450775426035781
   ],
   "max_abs_delta_p": 0.0005450775426037169,
   "mean_abs_delta_p": 0.0005450775426036475,
   "finite": true
  },
  {
   "id": "request-15-q05",
   "request_id": "request-15",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-15-q05-option-00",
    "request-15-q05-option-01",
    "request-15-q05-option-02",
    "request-15-q05-option-03",
    "request-15-q05-option-04"
   ],
   "options": [
    "very low",
    "low",
    "medium",
    "high",
    "critical"
   ],
   "nopts": 5,
   "scalars": [
    -6.4296875,
    -2.8125,
    -0.83984375,
    1.4873046875,
    -0.7861328125
   ],
   "scalars_oracle": [
    -6.44263219833374,
    -2.8297576904296875,
    -0.8664757013320923,
    1.476930022239685,
    -0.7765030860900879
   ],
   "p": [
    0.006638105123733906,
    0.052446354888491795,
    0.16190714281609525,
    0.6120549619234562,
    0.16695343524822281
   ],
   "p_oracle": [
    0.006626994094001005,
    0.052229686529919314,
    0.16037686679369648,
    0.61192850142172,
    0.16883795116066314
   ],
   "argmax": 3,
   "oracle_argmax": 3,
   "argmax_agrees": true,
   "selected_option": "high",
   "oracle_selected_option": "high",
   "oracle_margin": 0.4430905502610568,
   "abs_delta_p": [
    1.1111029732901317e-05,
    0.00021666835857248046,
    0.0015302760223987744,
    0.00012646050173625856,
    0.001884515912440321
   ],
   "max_abs_delta_p": 0.001884515912440321,
   "mean_abs_delta_p": 0.0007538063649761472,
   "finite": true
  },
  {
   "id": "request-16-q01",
   "request_id": "request-16",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-16-q01-option-00",
    "request-16-q01-option-01"
   ],
   "options": [
    "a minor visible blemish",
    "a completely unreadable page"
   ],
   "nopts": 2,
   "scalars": [
    3.55078125,
    -8.203125
   ],
   "scalars_oracle": [
    3.515467405319214,
    -8.229701042175293
   ],
   "p": [
    0.9987907172595901,
    0.001209282740409958
   ],
   "p_oracle": [
    0.9987846715320305,
    0.0012153284679694153
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "a minor visible blemish",
   "oracle_selected_option": "a minor visible blemish",
   "oracle_margin": 0.9975693430640611,
   "abs_delta_p": [
    6.045727559578218e-06,
    6.045727559457221e-06
   ],
   "max_abs_delta_p": 6.045727559578218e-06,
   "mean_abs_delta_p": 6.045727559517719e-06,
   "finite": true
  },
  {
   "id": "request-16-q02",
   "request_id": "request-16",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-16-q02-option-00",
    "request-16-q02-option-01"
   ],
   "options": [
    "all of the text",
    "none of the text"
   ],
   "nopts": 2,
   "scalars": [
    3.33203125,
    -5.1171875
   ],
   "scalars_oracle": [
    3.330043077468872,
    -5.142807960510254
   ],
   "p": [
    0.9920620058985926,
    0.007937994101407319
   ],
   "p_oracle": [
    0.992167647375149,
    0.007832352624851022
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "all of the text",
   "oracle_selected_option": "all of the text",
   "oracle_margin": 0.9843352947502979,
   "abs_delta_p": [
    0.0001056414765563396,
    0.00010564147655629623
   ],
   "max_abs_delta_p": 0.0001056414765563396,
   "mean_abs_delta_p": 0.00010564147655631791,
   "finite": true
  },
  {
   "id": "request-16-q03",
   "request_id": "request-16",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-16-q03-option-00",
    "request-16-q03-option-01"
   ],
   "options": [
    "keep the readable copy",
    "report a missing parcel"
   ],
   "nopts": 2,
   "scalars": [
    -0.66943359375,
    -13.1875
   ],
   "scalars_oracle": [
    -0.6904910802841187,
    -13.2025728225708
   ],
   "p": [
    0.9992182400776541,
    0.0007817599223458169
   ],
   "p_oracle": [
    0.9992155641329983,
    0.0007844358670016086
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "keep the readable copy",
   "oracle_selected_option": "keep the readable copy",
   "oracle_margin": 0.9984311282659967,
   "abs_delta_p": [
    2.6759446558610733e-06,
    2.6759446557916844e-06
   ],
   "max_abs_delta_p": 2.6759446558610733e-06,
   "mean_abs_delta_p": 2.675944655826379e-06,
   "finite": true
  },
  {
   "id": "request-16-q04",
   "request_id": "request-16",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-16-q04-option-00",
    "request-16-q04-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    2.2109375,
    -2.6953125
   ],
   "scalars_oracle": [
    2.1954517364501953,
    -2.713536024093628
   ],
   "p": [
    0.9428685125257794,
    0.0571314874742206
   ],
   "p_oracle": [
    0.942952726312319,
    0.057047273687681144
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.8859054526246378,
   "abs_delta_p": [
    8.421378653955536e-05,
    8.421378653945821e-05
   ],
   "max_abs_delta_p": 8.421378653955536e-05,
   "mean_abs_delta_p": 8.421378653950679e-05,
   "finite": true
  },
  {
   "id": "request-16-q05",
   "request_id": "request-16",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-16-q05-option-00",
    "request-16-q05-option-01",
    "request-16-q05-option-02",
    "request-16-q05-option-03",
    "request-16-q05-option-04"
   ],
   "options": [
    "very low",
    "low",
    "medium",
    "high",
    "critical"
   ],
   "nopts": 5,
   "scalars": [
    -2.46875,
    -0.1781005859375,
    -1.2763671875,
    -3.671875,
    -7.6796875
   ],
   "scalars_oracle": [
    -2.494438886642456,
    -0.20091819763183594,
    -1.2986819744110107,
    -3.687429666519165,
    -7.6829833984375
   ],
   "p": [
    0.13826351040868382,
    0.5118869922542627,
    0.27328712085284906,
    0.06952325911460011,
    0.007039117369604225
   ],
   "p_oracle": [
    0.13800648634764923,
    0.5117744176558293,
    0.27330553650463313,
    0.0697970447117941,
    0.007116514780093998
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "low",
   "oracle_selected_option": "low",
   "oracle_margin": 0.2384688811511962,
   "abs_delta_p": [
    0.00025702406103458864,
    0.00011257459843339035,
    1.8415651784065012e-05,
    0.0002737855971939873,
    7.739741048977313e-05
   ],
   "max_abs_delta_p": 0.0002737855971939873,
   "mean_abs_delta_p": 0.0001478394637871609,
   "finite": true
  },
  {
   "id": "request-17-q01",
   "request_id": "request-17",
   "type": "choice",
   "zoo_only": true,
   "row_ids": [
    "request-17-q01-option-00",
    "request-17-q01-option-01",
    "request-17-q01-option-02",
    "request-17-q01-option-03",
    "request-17-q01-option-04",
    "request-17-q01-option-05",
    "request-17-q01-option-06",
    "request-17-q01-option-07",
    "request-17-q01-option-08",
    "request-17-q01-option-09",
    "request-17-q01-option-10",
    "request-17-q01-option-11",
    "request-17-q01-option-12",
    "request-17-q01-option-13",
    "request-17-q01-option-14",
    "request-17-q01-option-15",
    "request-17-q01-option-16",
    "request-17-q01-option-17",
    "request-17-q01-option-18",
    "request-17-q01-option-19"
   ],
   "options": [
    "lighting",
    "plumbing",
    "heating",
    "cooling",
    "doors",
    "windows",
    "floors",
    "walls",
    "ceilings",
    "roofs",
    "stairs",
    "lifts",
    "locks",
    "shelves",
    "desks",
    "chairs",
    "curtains",
    "carpets",
    "drains",
    "pipes"
   ],
   "nopts": 20,
   "scalars": [
    5.5390625,
    -7.296875,
    -7.6171875,
    -7.90625,
    -8.3359375,
    -8.5078125,
    -8.5390625,
    -6.640625,
    2.853515625,
    -5.97265625,
    -8.578125,
    -8.8828125,
    -8.8515625,
    -9.96875,
    -9.90625,
    -9.7265625,
    -10.671875,
    -10.46875,
    -9.171875,
    -7.59375
   ],
   "scalars_oracle": [
    5.513814926147461,
    -7.31196403503418,
    -7.631340503692627,
    -7.937613487243652,
    -8.349449157714844,
    -8.52861499786377,
    -8.547520637512207,
    -6.649753093719482,
    2.8335750102996826,
    -5.990054130554199,
    -8.59441089630127,
    -8.899070739746094,
    -8.87210464477539,
    -9.984298706054688,
    -9.920188903808594,
    -9.742130279541016,
    -10.690881729125977,
    -10.486421585083008,
    -9.18571662902832,
    -7.604816436767578
   ],
   "p": [
    0.8177828640344648,
    0.0005335382179177008,
    0.0004442977696897327,
    0.0003766500167230497,
    0.00029464778810066176,
    0.00026708485547346405,
    0.00026235781438986203,
    0.0007762935264409497,
    0.1762667794455782,
    0.0011370894162933572,
    0.0002565664889610634,
    0.0002155689268090025,
    0.00021945294747651697,
    0.00011590199810471329,
    0.00012011616014916373,
    0.00013310494100881372,
    7.755282180048426e-05,
    8.70977197977468e-05,
    0.00018274689954955481,
    0.00045028821127106734
   ],
   "p_oracle": [
    0.8173185076199884,
    0.0005363396301957823,
    0.00044686956878130105,
    0.00037512287018886067,
    0.0002964619730113619,
    0.00026761208213463506,
    0.0002647365695935582,
    0.0007830322250443364,
    0.17670173583366977,
    0.0011415527626462557,
    0.00025773729274020525,
    0.00021655606655843212,
    0.000219918864143094,
    0.00011647995701225297,
    0.00012082622884148439,
    0.00013376722645384417,
    7.778569053030336e-05,
    8.742592434916856e-05,
    0.0001838374293344633,
    0.00045369418478281734
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "lighting",
   "oracle_selected_option": "lighting",
   "oracle_margin": 0.6406167717863186,
   "abs_delta_p": [
    0.00046435641447639675,
    2.8014122780815496e-06,
    2.5717990915683463e-06,
    1.5271465341890533e-06,
    1.8141849107001346e-06,
    5.272266611710099e-07,
    2.3787552036961656e-06,
    6.738698603386676e-06,
    0.000434956388091573,
    4.463346352898525e-06,
    1.170803779141868e-06,
    9.871397494296214e-07,
    4.659166665770388e-07,
    5.779589075396806e-07,
    7.100686923206559e-07,
    6.622854450304475e-07,
    2.328687298190998e-07,
    3.2820455142176796e-07,
    1.0905297849084933e-06,
    3.4059735117499988e-06
   ],
   "max_abs_delta_p": 0.00046435641447639675,
   "mean_abs_delta_p": 4.6588356101079995e-05,
   "finite": true
  },
  {
   "id": "request-18-q01",
   "request_id": "request-18",
   "type": "choice",
   "zoo_only": true,
   "row_ids": [
    "request-18-q01-option-00",
    "request-18-q01-option-01",
    "request-18-q01-option-02",
    "request-18-q01-option-03",
    "request-18-q01-option-04",
    "request-18-q01-option-05",
    "request-18-q01-option-06",
    "request-18-q01-option-07",
    "request-18-q01-option-08",
    "request-18-q01-option-09",
    "request-18-q01-option-10",
    "request-18-q01-option-11",
    "request-18-q01-option-12",
    "request-18-q01-option-13",
    "request-18-q01-option-14",
    "request-18-q01-option-15",
    "request-18-q01-option-16",
    "request-18-q01-option-17",
    "request-18-q01-option-18",
    "request-18-q01-option-19",
    "request-18-q01-option-20",
    "request-18-q01-option-21",
    "request-18-q01-option-22",
    "request-18-q01-option-23"
   ],
   "options": [
    "lighting",
    "plumbing",
    "heating",
    "cooling",
    "doors",
    "windows",
    "floors",
    "walls",
    "ceilings",
    "roofs",
    "stairs",
    "lifts",
    "locks",
    "shelves",
    "desks",
    "chairs",
    "curtains",
    "carpets",
    "drains",
    "pipes",
    "valves",
    "tanks",
    "pumps",
    "fans"
   ],
   "nopts": 24,
   "scalars": [
    -8.046875,
    3.837890625,
    -6.828125,
    -8.1640625,
    -8.8984375,
    -8.375,
    -6.5,
    -5.9921875,
    -5.3203125,
    -6.57421875,
    -9.8671875,
    -9.4453125,
    -9.2265625,
    -9.84375,
    -9.6640625,
    -10.7890625,
    -11.6953125,
    -10.6328125,
    -3.943359375,
    1.884765625,
    -5.1796875,
    -6.12890625,
    -6.7578125,
    -8.1640625
   ],
   "scalars_oracle": [
    -8.074076652526855,
    3.807345151901245,
    -6.854000091552734,
    -8.165847778320312,
    -8.922479629516602,
    -8.393821716308594,
    -6.5011138916015625,
    -6.00640869140625,
    -5.335151672363281,
    -6.5979790687561035,
    -9.877313613891602,
    -9.465591430664062,
    -9.240874290466309,
    -9.860201835632324,
    -9.681700706481934,
    -10.806360244750977,
    -11.715137481689453,
    -10.642301559448242,
    -3.9700522422790527,
    1.8557240962982178,
    -5.19008207321167,
    -6.140894889831543,
    -6.777218341827393,
    -8.173320770263672
   ],
   "p": [
    0.0008170248407383827,
    0.7272053232907862,
    0.001639420444147085,
    0.0007641049994084376,
    0.0005022320964456775,
    0.0006773373832488235,
    0.001977518531249393,
    0.0026432820957597463,
    0.003880445375400284,
    0.0018954041306972764,
    0.000288729691758335,
    0.00036744085111829484,
    0.0004163650320382538,
    0.00029262261778513656,
    0.0003242654129949427,
    0.00017049487088822057,
    0.00010158014274781289,
    0.00018641790001974848,
    0.008523101425365462,
    0.23820637850988804,
    0.004205137858786907,
    0.002444636309259044,
    0.0017066311900605106,
    0.0007641049994084376
   ],
   "p_oracle": [
    0.0008182089163662708,
    0.7268690316279388,
    0.0016430413896767416,
    0.0007764071494108794,
    0.0005038688428868821,
    0.0006815749601400866,
    0.0020101277703289343,
    0.0026668205916226774,
    0.0039136185745641825,
    0.001901887392480624,
    0.0002919832922165861,
    0.00036943189294090403,
    0.0004200510265068098,
    0.00029485235549343234,
    0.00032651483575018203,
    0.0001717109934837397,
    0.00010215706837264009,
    0.00018858722148271857,
    0.008537935465879583,
    0.23830092850425097,
    0.004251871872072085,
    0.0024695543690186777,
    0.0017167351455453683,
    0.000773098741569937
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "plumbing",
   "oracle_selected_option": "plumbing",
   "oracle_margin": 0.4885681031236878,
   "abs_delta_p": [
    1.1840756278880607e-06,
    0.00033629166284743217,
    3.6209455296567095e-06,
    1.2302150002441842e-05,
    1.6367464412045916e-06,
    4.237576891263093e-06,
    3.260923907954117e-05,
    2.3538495862931042e-05,
    3.317319916389854e-05,
    6.483261783347656e-06,
    3.2536004582510913e-06,
    1.9910418226091933e-06,
    3.6859944685559685e-06,
    2.2297377082957856e-06,
    2.249422755239356e-06,
    1.2161225955191143e-06,
    5.769256248271956e-07,
    2.169321462970093e-06,
    1.48340405141216e-05,
    9.454999436292466e-05,
    4.673401328517731e-05,
    2.4918059759633515e-05,
    1.0103955484857706e-05,
    8.993742161499466e-06
   ],
   "max_abs_delta_p": 0.00033629166284743217,
   "mean_abs_delta_p": 2.802430523725362e-05,
   "finite": true
  },
  {
   "id": "request-19-q01",
   "request_id": "request-19",
   "type": "choice",
   "zoo_only": true,
   "row_ids": [
    "request-19-q01-option-00",
    "request-19-q01-option-01",
    "request-19-q01-option-02",
    "request-19-q01-option-03",
    "request-19-q01-option-04",
    "request-19-q01-option-05",
    "request-19-q01-option-06",
    "request-19-q01-option-07",
    "request-19-q01-option-08",
    "request-19-q01-option-09",
    "request-19-q01-option-10",
    "request-19-q01-option-11",
    "request-19-q01-option-12",
    "request-19-q01-option-13",
    "request-19-q01-option-14",
    "request-19-q01-option-15",
    "request-19-q01-option-16",
    "request-19-q01-option-17",
    "request-19-q01-option-18",
    "request-19-q01-option-19",
    "request-19-q01-option-20",
    "request-19-q01-option-21",
    "request-19-q01-option-22",
    "request-19-q01-option-23",
    "request-19-q01-option-24",
    "request-19-q01-option-25",
    "request-19-q01-option-26",
    "request-19-q01-option-27"
   ],
   "options": [
    "lighting",
    "plumbing",
    "heating",
    "cooling",
    "doors",
    "windows",
    "floors",
    "walls",
    "ceilings",
    "roofs",
    "stairs",
    "lifts",
    "locks",
    "shelves",
    "desks",
    "chairs",
    "curtains",
    "carpets",
    "drains",
    "pipes",
    "valves",
    "tanks",
    "pumps",
    "fans",
    "sockets",
    "switches",
    "cables",
    "sensors"
   ],
   "nopts": 28,
   "scalars": [
    -7.609375,
    -8.2578125,
    -7.6015625,
    -1.0126953125,
    -9.1953125,
    -8.703125,
    -10.09375,
    -9.6796875,
    -5.94921875,
    -9.609375,
    -10.5078125,
    -10.109375,
    -8.015625,
    -9.9296875,
    -10.59375,
    -9.671875,
    -11.84375,
    -11.140625,
    -9.375,
    -8.1953125,
    -8.9140625,
    -7.99609375,
    -6.95703125,
    1.2744140625,
    -7.11328125,
    -7.0390625,
    -7.69140625,
    -7.49609375
   ],
   "scalars_oracle": [
    -7.635459899902344,
    -8.273568153381348,
    -7.611283779144287,
    -1.0317230224609375,
    -9.20922565460205,
    -8.708393096923828,
    -10.114972114562988,
    -9.697352409362793,
    -5.983921051025391,
    -9.618802070617676,
    -10.518702507019043,
    -10.130147933959961,
    -8.015636444091797,
    -9.947691917419434,
    -10.613463401794434,
    -9.693241119384766,
    -11.859024047851562,
    -11.160334587097168,
    -9.379319190979004,
    -8.208106994628906,
    -8.927600860595703,
    -7.999220848083496,
    -6.959987640380859,
    1.2542405128479004,
    -7.119679927825928,
    -7.052636623382568,
    -7.682580471038818,
    -7.514122486114502
   ],
   "p": [
    0.004516916146606102,
    0.0031183180759876685,
    0.004537126028610805,
    0.19585228801753204,
    0.0018249990975535485,
    0.0024177309891328753,
    0.0010921919446881111,
    0.0013837452581967364,
    0.011663802081824624,
    0.001440474178035206,
    0.0008620685324668325,
    0.0010824836360547442,
    0.0035811595169917736,
    0.001199538183214859,
    0.0008207574461128541,
    0.0013899365018428546,
    0.0004017949622638131,
    0.0006004789751504953,
    0.0016469101914053355,
    0.003231699190146536,
    0.002143186581486417,
    0.003621351684623212,
    0.006557420765728099,
    0.7236301254548227,
    0.005997313598608888,
    0.006257134606222532,
    0.004310071496715403,
    0.004818976857974617
   ],
   "p_oracle": [
    0.004499781253382536,
    0.003124878797177552,
    0.004562376779827329,
    0.19589772566343958,
    0.0018307652855539633,
    0.0024373809976546406,
    0.0010910763243614134,
    0.0013851445384904116,
    0.011562478811383397,
    0.0014487344786067763,
    0.0008662875177938502,
    0.001081655530193219,
    0.0036211260670016426,
    0.0012005182597137926,
    0.0008206262574979196,
    0.0013884024958851027,
    0.00040275113364651385,
    0.0006003843042419653,
    0.0016611958951607308,
    0.0032439829291755987,
    0.002150418602406555,
    0.003655253274956507,
    0.00661945429342202,
    0.7233242444573894,
    0.006042151746046325,
    0.006278120533185768,
    0.0043802366148561886,
    0.004822847157549282
   ],
   "argmax": 23,
   "oracle_argmax": 23,
   "argmax_agrees": true,
   "selected_option": "fans",
   "oracle_selected_option": "fans",
   "oracle_margin": 0.5274265187939499,
   "abs_delta_p": [
    1.713489322356654e-05,
    6.560721189883493e-06,
    2.525075121652398e-05,
    4.543764590753119e-05,
    5.766188000414863e-06,
    1.9650008521765287e-05,
    1.1156203266976614e-06,
    1.399280293675224e-06,
    0.00010132327044122709,
    8.260300571570406e-06,
    4.218985327017737e-06,
    8.281058615253016e-07,
    3.996655000986903e-05,
    9.800764989335246e-07,
    1.3118861493454596e-07,
    1.5340059577519539e-06,
    9.561713827007648e-07,
    9.467090852997417e-08,
    1.4285703755395271e-05,
    1.2283739029062858e-05,
    7.232020920138263e-06,
    3.390159033329508e-05,
    6.203352769392021e-05,
    0.00030588099743333874,
    4.483814743743743e-05,
    2.098592696323561e-05,
    7.016511814078585e-05,
    3.87029957466499e-06
   ],
   "max_abs_delta_p": 0.00030588099743333874,
   "mean_abs_delta_p": 3.0574482340549746e-05,
   "finite": true
  },
  {
   "id": "request-20-q01",
   "request_id": "request-20",
   "type": "choice",
   "zoo_only": true,
   "row_ids": [
    "request-20-q01-option-00",
    "request-20-q01-option-01",
    "request-20-q01-option-02",
    "request-20-q01-option-03",
    "request-20-q01-option-04",
    "request-20-q01-option-05",
    "request-20-q01-option-06",
    "request-20-q01-option-07",
    "request-20-q01-option-08",
    "request-20-q01-option-09",
    "request-20-q01-option-10",
    "request-20-q01-option-11",
    "request-20-q01-option-12",
    "request-20-q01-option-13",
    "request-20-q01-option-14",
    "request-20-q01-option-15",
    "request-20-q01-option-16",
    "request-20-q01-option-17",
    "request-20-q01-option-18",
    "request-20-q01-option-19",
    "request-20-q01-option-20",
    "request-20-q01-option-21",
    "request-20-q01-option-22",
    "request-20-q01-option-23",
    "request-20-q01-option-24",
    "request-20-q01-option-25",
    "request-20-q01-option-26",
    "request-20-q01-option-27",
    "request-20-q01-option-28",
    "request-20-q01-option-29",
    "request-20-q01-option-30",
    "request-20-q01-option-31"
   ],
   "options": [
    "lighting",
    "plumbing",
    "heating",
    "cooling",
    "doors",
    "windows",
    "floors",
    "walls",
    "ceilings",
    "roofs",
    "stairs",
    "lifts",
    "locks",
    "shelves",
    "desks",
    "chairs",
    "curtains",
    "carpets",
    "drains",
    "pipes",
    "valves",
    "tanks",
    "pumps",
    "fans",
    "sockets",
    "switches",
    "cables",
    "sensors",
    "meters",
    "signs",
    "paths",
    "gates"
   ],
   "nopts": 32,
   "scalars": [
    -7.78515625,
    -7.390625,
    -8.3125,
    -8.359375,
    2.419921875,
    -5.3359375,
    -8.9140625,
    -6.8828125,
    -8.859375,
    -8.1484375,
    -9.203125,
    -6.06640625,
    -1.203125,
    -9.8359375,
    -10.5,
    -10.265625,
    -6.25,
    -10.7421875,
    -8.1015625,
    -7.09375,
    -2.53515625,
    -7.37890625,
    -6.9140625,
    -8.125,
    -7.984375,
    -4.9453125,
    -5.60546875,
    -3.224609375,
    -8.1953125,
    -9.203125,
    -7.89453125,
    4.1953125
   ],
   "scalars_oracle": [
    -7.80185079574585,
    -7.406404495239258,
    -8.328375816345215,
    -8.3722505569458,
    2.405498504638672,
    -5.341571807861328,
    -8.924745559692383,
    -6.8783721923828125,
    -8.876729011535645,
    -8.154253005981445,
    -9.206731796264648,
    -6.098209381103516,
    -1.189605712890625,
    -9.841769218444824,
    -10.507135391235352,
    -10.270621299743652,
    -6.233147144317627,
    -10.75239372253418,
    -8.108002662658691,
    -7.108526229858398,
    -2.5550146102905273,
    -7.36898946762085,
    -6.927596569061279,
    -8.134787559509277,
    -8.004584312438965,
    -4.9726715087890625,
    -5.6035261154174805,
    -3.2340099811553955,
    -8.21119499206543,
    -9.236793518066406,
    -7.896141052246094,
    4.17133903503418
   ],
   "p": [
    0.000717858944322546,
    0.0008993924878897198,
    0.000531091226422738,
    0.0005170544010148663,
    0.2446897370635693,
    0.0029097622762010423,
    0.0003766000720946237,
    0.0012021875005124253,
    0.0003885546416473148,
    0.0005832896021279521,
    0.0003192598143164164,
    0.0019168067904573286,
    0.0308667183897491,
    0.00022238232679585153,
    0.00015216018394047576,
    0.0001739664351356079,
    0.0017259022500446,
    0.00013249447553952243,
    0.0005991245593224508,
    0.0010656736134457185,
    0.014418582331831017,
    0.0009054354158788976,
    0.0011809104060285715,
    0.0005911540626877878,
    0.0006406183025131938,
    0.0036374601297118397,
    0.002494413243690419,
    0.009723489927841967,
    0.0005678731653653913,
    0.0003192598143164164,
    0.0006743660695540914,
    0.6748564200760307
   ],
   "p_oracle": [
    0.0007191815249930839,
    0.0009015207946759931,
    0.0005323186915273457,
    0.0005191386905016756,
    0.2454589060345332,
    0.0029336055303064035,
    0.0003785922027948519,
    0.0012190362342608175,
    0.00038912384915785244,
    0.0005880083232525815,
    0.0003222490363386637,
    0.0019038304734785964,
    0.03146211983276343,
    0.00022417928797102953,
    0.00015327548882480683,
    0.00017545591105207873,
    0.0017625481986839066,
    0.0001332316396700548,
    0.0006037558356560152,
    0.0010688079711317084,
    0.014419055372251894,
    0.0009210028446555952,
    0.001185224679289517,
    0.0005945853163533855,
    0.0006405108561773736,
    0.003622021856556868,
    0.002525765151669818,
    0.009782091018493599,
    0.0005691834701096167,
    0.0003167606763826054,
    0.0006814573372852584,
    0.6732934558692004
   ],
   "argmax": 31,
   "oracle_argmax": 31,
   "argmax_agrees": true,
   "selected_option": "gates",
   "oracle_selected_option": "gates",
   "oracle_margin": 0.42783454983466723,
   "abs_delta_p": [
    1.3225806705379102e-06,
    2.128306786273364e-06,
    1.2274651046077344e-06,
    2.084289486809377e-06,
    0.0007691689709638794,
    2.3843254105361273e-05,
    1.992130700228178e-06,
    1.6848733748392204e-05,
    5.692075105376304e-07,
    4.7187211246294295e-06,
    2.989222022247329e-06,
    1.2976316978732154e-05,
    0.0005954014430143324,
    1.7969611751780042e-06,
    1.115304884331071e-06,
    1.4894759164708362e-06,
    3.664594863930673e-05,
    7.371641305323835e-07,
    4.6312763335643785e-06,
    3.1343576859898907e-06,
    4.7304042087754905e-07,
    1.55674287766976e-05,
    4.3142732609453924e-06,
    3.4312536655976703e-06,
    1.0744633582014142e-07,
    1.543827315497173e-05,
    3.135190797939891e-05,
    5.860109065163217e-05,
    1.3103047442254321e-06,
    2.499137933811011e-06,
    7.091267731167074e-06,
    0.0015629642068303307
   ],
   "max_abs_delta_p": 0.0015629642068303307,
   "mean_abs_delta_p": 9.962408632710677e-05,
   "finite": true
  }
 ],
 "sessions": [
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-000-rows-a1",
   "mode": "fp16",
   "indices": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    12,
    13
   ],
   "pid": 7907,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:42:52+00:00",
   "phase": "complete",
   "runtime_calls": 5045,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 5045,
   "result": "PASS",
   "load_wall_seconds_contended": 12.24357608309947,
   "active_row": "request-01-q01-option-00",
   "active_step": 383,
   "reset_check": {
    "row": "request-01-q01-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 4.3125,
    "final_scalar": 4.3125,
    "scalar_dtype": "<f2",
    "first_bits": "5044",
    "final_bits": "5044",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:45:20+00:00",
   "wall_seconds_contended": 148.33977275004145,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-000-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-000-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-000-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 149.15001662506256
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-001-rows-a1",
   "mode": "fp16",
   "indices": [
    14,
    15,
    16,
    17,
    18,
    19,
    20,
    21,
    22,
    23,
    24,
    25,
    26,
    27
   ],
   "pid": 14282,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:45:21+00:00",
   "phase": "complete",
   "runtime_calls": 456,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 456,
   "result": "PASS",
   "load_wall_seconds_contended": 19.27490279194899,
   "active_row": "request-07-q01-option-02",
   "active_step": 25,
   "reset_check": {
    "row": "request-07-q01-option-02",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -9.296875,
    "final_scalar": -9.296875,
    "scalar_dtype": "<f2",
    "first_bits": "a6c8",
    "final_bits": "a6c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:46:16+00:00",
   "wall_seconds_contended": 55.3243713749107,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-001-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-001-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-001-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 56.55378474993631
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-002-rows-a1",
   "mode": "fp16",
   "indices": [
    28,
    29,
    30,
    31,
    32,
    33,
    34,
    35,
    36,
    37,
    38,
    39,
    40,
    41
   ],
   "pid": 16627,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:46:18+00:00",
   "phase": "complete",
   "runtime_calls": 465,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 465,
   "result": "PASS",
   "load_wall_seconds_contended": 1.8712207910139114,
   "active_row": "request-07-q03-option-05",
   "active_step": 37,
   "reset_check": {
    "row": "request-07-q03-option-05",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 0.7021484375,
    "final_scalar": 0.7021484375,
    "scalar_dtype": "<f2",
    "first_bits": "9e39",
    "final_bits": "9e39",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:46:46+00:00",
   "wall_seconds_contended": 28.230316624976695,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-002-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-002-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-002-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 29.34776079200674
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-003-rows-a1",
   "mode": "fp16",
   "indices": [
    42,
    43,
    44,
    45,
    46,
    47,
    48,
    49,
    50,
    51,
    52,
    53,
    54,
    55
   ],
   "pid": 17813,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:46:47+00:00",
   "phase": "complete",
   "runtime_calls": 553,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 553,
   "result": "PASS",
   "load_wall_seconds_contended": 1.8364604170201346,
   "active_row": "request-08-q01-option-09",
   "active_step": 26,
   "reset_check": {
    "row": "request-08-q01-option-09",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -10.6953125,
    "final_scalar": -10.6953125,
    "scalar_dtype": "<f2",
    "first_bits": "59c9",
    "final_bits": "59c9",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:47:18+00:00",
   "wall_seconds_contended": 30.94646545802243,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-003-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-003-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-003-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 32.01531366596464
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-004-rows-a1",
   "mode": "fp16",
   "indices": [
    56,
    57,
    58,
    59,
    60,
    61,
    62,
    63,
    64,
    65,
    66,
    67,
    68,
    69
   ],
   "pid": 19147,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:47:19+00:00",
   "phase": "complete",
   "runtime_calls": 398,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 398,
   "result": "PASS",
   "load_wall_seconds_contended": 1.8380655420478433,
   "active_row": "request-09-q01-option-01",
   "active_step": 25,
   "reset_check": {
    "row": "request-09-q01-option-01",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -7.54296875,
    "final_scalar": -7.54296875,
    "scalar_dtype": "<f2",
    "first_bits": "8bc7",
    "final_bits": "8bc7",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:47:43+00:00",
   "wall_seconds_contended": 24.564215915976092,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-004-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-004-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-004-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 25.56546025001444
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-005-rows-a1",
   "mode": "fp16",
   "indices": [
    70,
    71,
    72,
    73,
    74,
    75,
    76,
    77,
    78,
    79,
    80,
    81,
    82,
    83
   ],
   "pid": 20178,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:47:44+00:00",
   "phase": "complete",
   "runtime_calls": 393,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 393,
   "result": "PASS",
   "load_wall_seconds_contended": 1.847309833043255,
   "active_row": "request-10-q01-option-00",
   "active_step": 25,
   "reset_check": {
    "row": "request-10-q01-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 0.86767578125,
    "final_scalar": 0.86767578125,
    "scalar_dtype": "<f2",
    "first_bits": "f13a",
    "final_bits": "f13a",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:48:09+00:00",
   "wall_seconds_contended": 24.30700445792172,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-005-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-005-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-005-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 25.269711875007488
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-006-rows-a1",
   "mode": "fp16",
   "indices": [
    84,
    85,
    86,
    87,
    88,
    89,
    90,
    91,
    92,
    93,
    94,
    95,
    96,
    97
   ],
   "pid": 21289,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:48:10+00:00",
   "phase": "complete",
   "runtime_calls": 410,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 410,
   "result": "PASS",
   "load_wall_seconds_contended": 1.8375859999796376,
   "active_row": "request-10-q03-option-00",
   "active_step": 29,
   "reset_check": {
    "row": "request-10-q03-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -2.599609375,
    "final_scalar": -2.599609375,
    "scalar_dtype": "<f2",
    "first_bits": "33c1",
    "final_bits": "33c1",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:48:35+00:00",
   "wall_seconds_contended": 25.048792415997013,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-006-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-006-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-006-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 26.025147167034447
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-007-rows-a1",
   "mode": "fp16",
   "indices": [
    98,
    99,
    100,
    101,
    102,
    103,
    104,
    105,
    106,
    107,
    108,
    109,
    110,
    111
   ],
   "pid": 22389,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:48:36+00:00",
   "phase": "complete",
   "runtime_calls": 407,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 407,
   "result": "PASS",
   "load_wall_seconds_contended": 1.879706917097792,
   "active_row": "request-11-q01-option-12",
   "active_step": 26,
   "reset_check": {
    "row": "request-11-q01-option-12",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -9.3125,
    "final_scalar": -9.3125,
    "scalar_dtype": "<f2",
    "first_bits": "a8c8",
    "final_bits": "a8c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:48:52+00:00",
   "wall_seconds_contended": 15.849513333057985,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-007-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-007-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-007-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 16.582962165935896
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-008-rows-a1",
   "mode": "fp16",
   "indices": [
    112,
    113,
    114,
    115,
    116,
    117,
    118,
    119,
    120,
    121,
    122,
    123,
    124,
    125
   ],
   "pid": 23081,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:48:52+00:00",
   "phase": "complete",
   "runtime_calls": 421,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 421,
   "result": "PASS",
   "load_wall_seconds_contended": 1.4225483750924468,
   "active_row": "request-12-q01-option-07",
   "active_step": 27,
   "reset_check": {
    "row": "request-12-q01-option-07",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -2.65234375,
    "final_scalar": -2.65234375,
    "scalar_dtype": "<f2",
    "first_bits": "4ec1",
    "final_bits": "4ec1",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:49:08+00:00",
   "wall_seconds_contended": 15.45467358303722,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-008-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-008-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-008-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 16.11218466598075
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-009-rows-a1",
   "mode": "fp16",
   "indices": [
    126,
    127,
    128,
    129,
    130,
    131,
    132,
    133,
    134,
    135,
    136,
    137,
    138,
    139
   ],
   "pid": 23783,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:49:08+00:00",
   "phase": "complete",
   "runtime_calls": 460,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 460,
   "result": "PASS",
   "load_wall_seconds_contended": 1.5216525830328465,
   "active_row": "request-13-q01-option-00",
   "active_step": 30,
   "reset_check": {
    "row": "request-13-q01-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 4.42578125,
    "final_scalar": 4.42578125,
    "scalar_dtype": "<f2",
    "first_bits": "6d44",
    "final_bits": "6d44",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:49:25+00:00",
   "wall_seconds_contended": 16.367724666954018,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-009-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-009-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-009-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 17.028749457909726
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-010-rows-a1",
   "mode": "fp16",
   "indices": [
    140,
    141,
    142,
    143,
    144,
    145,
    146,
    147,
    148,
    149,
    150,
    151,
    152,
    153
   ],
   "pid": 24468,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:49:25+00:00",
   "phase": "complete",
   "runtime_calls": 455,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 455,
   "result": "PASS",
   "load_wall_seconds_contended": 1.4069495829753578,
   "active_row": "request-14-q02-option-00",
   "active_step": 31,
   "reset_check": {
    "row": "request-14-q02-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -0.386962890625,
    "final_scalar": -0.386962890625,
    "scalar_dtype": "<f2",
    "first_bits": "31b6",
    "final_bits": "31b6",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:49:42+00:00",
   "wall_seconds_contended": 16.428511666948907,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-010-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-010-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-010-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 17.157692458946258
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-011-rows-a1",
   "mode": "fp16",
   "indices": [
    154,
    155,
    156,
    157,
    158,
    159,
    160,
    161,
    162,
    163,
    164,
    165,
    166,
    167
   ],
   "pid": 25169,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:49:43+00:00",
   "phase": "complete",
   "runtime_calls": 452,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 452,
   "result": "PASS",
   "load_wall_seconds_contended": 14.065486541949213,
   "active_row": "request-15-q03-option-00",
   "active_step": 29,
   "reset_check": {
    "row": "request-15-q03-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -2.009765625,
    "final_scalar": -2.009765625,
    "scalar_dtype": "<f2",
    "first_bits": "05c0",
    "final_bits": "05c0",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:50:39+00:00",
   "wall_seconds_contended": 56.24541620898526,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-011-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-011-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-011-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 57.351253666915
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-012-rows-a1",
   "mode": "fp16",
   "indices": [
    168,
    169,
    170,
    171,
    172,
    173,
    174,
    175,
    176,
    177,
    178,
    179,
    180,
    181
   ],
   "pid": 27540,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:50:40+00:00",
   "phase": "complete",
   "runtime_calls": 437,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 437,
   "result": "PASS",
   "load_wall_seconds_contended": 2.162571999942884,
   "active_row": "request-16-q03-option-01",
   "active_step": 33,
   "reset_check": {
    "row": "request-16-q03-option-01",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -13.1875,
    "final_scalar": -13.1875,
    "scalar_dtype": "<f2",
    "first_bits": "98ca",
    "final_bits": "98ca",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:51:04+00:00",
   "wall_seconds_contended": 23.568665999919176,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-012-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-012-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-012-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 24.36194324993994
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-013-rows-a1",
   "mode": "fp16",
   "indices": [
    182,
    183,
    184,
    185,
    186,
    187,
    188,
    189,
    190,
    191,
    192,
    193,
    194,
    195
   ],
   "pid": 28571,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:51:04+00:00",
   "phase": "complete",
   "runtime_calls": 387,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 387,
   "result": "PASS",
   "load_wall_seconds_contended": 1.4953227499499917,
   "active_row": "request-17-q01-option-06",
   "active_step": 25,
   "reset_check": {
    "row": "request-17-q01-option-06",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -8.5390625,
    "final_scalar": -8.5390625,
    "scalar_dtype": "<f2",
    "first_bits": "45c8",
    "final_bits": "45c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:51:21+00:00",
   "wall_seconds_contended": 16.46153820795007,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-013-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-013-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-013-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 17.92508495808579
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-014-rows-a1",
   "mode": "fp16",
   "indices": [
    196,
    197,
    198,
    199,
    200,
    201,
    202,
    203,
    204,
    205,
    206,
    207,
    208,
    209
   ],
   "pid": 29300,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:51:23+00:00",
   "phase": "complete",
   "runtime_calls": 415,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 415,
   "result": "PASS",
   "load_wall_seconds_contended": 1.658332291059196,
   "active_row": "request-18-q01-option-00",
   "active_step": 27,
   "reset_check": {
    "row": "request-18-q01-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -8.046875,
    "final_scalar": -8.046875,
    "scalar_dtype": "<f2",
    "first_bits": "06c8",
    "final_bits": "06c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:51:44+00:00",
   "wall_seconds_contended": 21.497437790967524,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-014-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-014-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-014-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 22.922678375034593
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-015-rows-a1",
   "mode": "fp16",
   "indices": [
    210,
    211,
    212,
    213,
    214,
    215,
    216,
    217,
    218,
    219,
    220,
    221,
    222,
    223
   ],
   "pid": 30237,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:51:45+00:00",
   "phase": "complete",
   "runtime_calls": 411,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 411,
   "result": "PASS",
   "load_wall_seconds_contended": 6.910803542006761,
   "active_row": "request-18-q01-option-14",
   "active_step": 27,
   "reset_check": {
    "row": "request-18-q01-option-14",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -9.6640625,
    "final_scalar": -9.6640625,
    "scalar_dtype": "<f2",
    "first_bits": "d5c8",
    "final_bits": "d5c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:52:16+00:00",
   "wall_seconds_contended": 31.100759833003394,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-015-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-015-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-015-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 32.20412429200951
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-016-rows-a1",
   "mode": "fp16",
   "indices": [
    224,
    225,
    226,
    227,
    228,
    229,
    230,
    231,
    232,
    233,
    234,
    235,
    236,
    237
   ],
   "pid": 31586,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:52:17+00:00",
   "phase": "complete",
   "runtime_calls": 385,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 385,
   "result": "PASS",
   "load_wall_seconds_contended": 1.7944538329029456,
   "active_row": "request-19-q01-option-04",
   "active_step": 24,
   "reset_check": {
    "row": "request-19-q01-option-04",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -9.1953125,
    "final_scalar": -9.1953125,
    "scalar_dtype": "<f2",
    "first_bits": "99c8",
    "final_bits": "99c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:52:42+00:00",
   "wall_seconds_contended": 24.130284834071063,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-016-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-016-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-016-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 25.13393245905172
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-017-rows-a1",
   "mode": "fp16",
   "indices": [
    238,
    239,
    240,
    241,
    242,
    243,
    244,
    245,
    246,
    247,
    248,
    249,
    250,
    251
   ],
   "pid": 32608,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:52:43+00:00",
   "phase": "complete",
   "runtime_calls": 383,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 383,
   "result": "PASS",
   "load_wall_seconds_contended": 3.460860832943581,
   "active_row": "request-19-q01-option-18",
   "active_step": 25,
   "reset_check": {
    "row": "request-19-q01-option-18",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -9.375,
    "final_scalar": -9.375,
    "scalar_dtype": "<f2",
    "first_bits": "b0c8",
    "final_bits": "b0c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:53:10+00:00",
   "wall_seconds_contended": 27.458986666053534,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-017-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-017-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-017-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 28.497000584029593
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-018-rows-a1",
   "mode": "fp16",
   "indices": [
    252,
    253,
    254,
    255,
    256,
    257,
    258,
    259,
    260,
    261,
    262,
    263,
    264,
    265
   ],
   "pid": 33804,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:53:11+00:00",
   "phase": "complete",
   "runtime_calls": 370,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 370,
   "result": "PASS",
   "load_wall_seconds_contended": 2.876136083039455,
   "active_row": "request-20-q01-option-04",
   "active_step": 23,
   "reset_check": {
    "row": "request-20-q01-option-04",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 2.419921875,
    "final_scalar": 2.419921875,
    "scalar_dtype": "<f2",
    "first_bits": "d740",
    "final_bits": "d740",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:53:30+00:00",
   "wall_seconds_contended": 18.43355870898813,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-018-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-018-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-018-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 19.215317790978588
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-019-rows-a1",
   "mode": "fp16",
   "indices": [
    266,
    267,
    268,
    269,
    270,
    271,
    272,
    273,
    274,
    275,
    276,
    277,
    278,
    279
   ],
   "pid": 34583,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:53:30+00:00",
   "phase": "complete",
   "runtime_calls": 371,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 371,
   "result": "PASS",
   "load_wall_seconds_contended": 2.7563546668970957,
   "active_row": "request-20-q01-option-18",
   "active_step": 24,
   "reset_check": {
    "row": "request-20-q01-option-18",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -8.1015625,
    "final_scalar": -8.1015625,
    "scalar_dtype": "<f2",
    "first_bits": "0dc8",
    "final_bits": "0dc8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:53:46+00:00",
   "wall_seconds_contended": 15.990237083053216,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-019-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-019-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-019-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 16.787547707906924
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "fp16-020-global_reset-a1",
   "mode": "fp16",
   "indices": [
    0
   ],
   "pid": 35260,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T00:53:47+00:00",
   "phase": "complete",
   "runtime_calls": 768,
   "row_executions": 2,
   "planned_row_executions": 2,
   "planned_runtime_calls": 768,
   "result": "PASS",
   "load_wall_seconds_contended": 1.3302383340196684,
   "active_row": "request-01-q01-option-00",
   "active_step": 383,
   "reset_check": {
    "row": "request-01-q01-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 4.3125,
    "final_scalar": 4.3125,
    "scalar_dtype": "<f2",
    "first_bits": "5044",
    "final_bits": "5044",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T00:54:09+00:00",
   "wall_seconds_contended": 22.208872459013946,
   "kind": "global_reset",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-020-global_reset-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-fp16-020-global_reset-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/fp16-020-global_reset-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 22.896464750054292
  }
 ],
 "result": "PASS",
 "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
 "aot": {
  "source": {
   "path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/exports/system_one_qwen3_5_4b_scorer_decode_fp16/system_one_qwen3_5_4b_scorer_decode_fp16.aimodel",
   "bytes": 160,
   "mtime_ns": 1790114339532808245
  },
  "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_fp16.h16c.aimodelc",
  "command": [
   "/var/run/com.apple.security.cryptexd/mnt/com.apple.MobileAsset.MetalToolchain-v27.1.266.1.9a6Ccy/Metal.xctoolchain/usr/bin/coreai-build",
   "compile",
   "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/exports/system_one_qwen3_5_4b_scorer_decode_fp16/system_one_qwen3_5_4b_scorer_decode_fp16.aimodel",
   "--output",
   "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc",
   "--platform",
   "macOS",
   "--preferred-compute",
   "gpu",
   "--architecture",
   "h16c",
   "--expect-frequent-reshapes"
  ],
  "flags": [
   "--platform",
   "macOS",
   "--preferred-compute",
   "gpu",
   "--architecture",
   "h16c",
   "--expect-frequent-reshapes"
  ],
  "license": "cc-by-nc-4.0",
  "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/aot-fp16.log",
  "started_at": "2026-09-23T00:42:05+00:00",
  "result": "PASS",
  "reused": false,
  "returncode": 0,
  "wall_seconds_contended": 46.91745175002143,
  "completed_at": "2026-09-23T00:42:52+00:00"
 },
 "summary": {
  "rows": 280,
  "expected_rows": 280,
  "questions": 48,
  "expected_questions": 48,
  "argmax_agreement": 48,
  "margin_ge_0_02_questions": 46,
  "margin_ge_0_02_argmax_agreement": 46,
  "near_ties": [
   {
    "id": "request-07-q03",
    "oracle_margin": 0.006739843342914137,
    "oracle_argmax": 1,
    "argmax": 1,
    "argmax_agrees": true,
    "p": [
     0.06852844678950806,
     0.14041521453796316,
     0.13387302790675262,
     0.11745231516189299,
     0.09834065836381406,
     0.11402942509386077,
     0.11824146549641584,
     0.11257522991562542,
     0.06747071791742688,
     0.029073498816740265
    ],
    "p_oracle": [
     0.06844557046678758,
     0.1410972557943501,
     0.13435741245143595,
     0.11769923201661937,
     0.09850107499890995,
     0.11399026048335885,
     0.11797473484160344,
     0.11195394081789756,
     0.0672594693782866,
     0.028721048750750485
    ]
   },
   {
    "id": "request-08-q03",
    "oracle_margin": 0.004176430865158717,
    "oracle_argmax": 5,
    "argmax": 5,
    "argmax_agrees": true,
    "p": [
     0.03821268801456236,
     0.08558664157346375,
     0.1121254686745306,
     0.11465641135757645,
     0.10849394805786149,
     0.1272676828719541,
     0.12314573609528758,
     0.1172444835308317,
     0.10855450845152134,
     0.06471243137241071
    ],
    "p_oracle": [
     0.03829370334805408,
     0.08559956498729611,
     0.11225994259558048,
     0.11473061408817004,
     0.10857444261658075,
     0.12713889005910925,
     0.12296245919395053,
     0.11719791061365697,
     0.10862090712691694,
     0.06462156537068481
    ]
   }
  ],
  "strict_disagreements": [],
  "max_abs_delta_p": 0.003684344586315391,
  "mean_of_question_mean_abs_delta_p": 0.00020850158130700705,
  "mean_of_row_mean_abs_delta_p": 0.00020850158130700705,
  "mean_of_row_mean_note": "Each probability row is one question; options are weighted equally within it.",
  "mean_option_abs_delta_p": 0.00013447082908294965,
  "max_abs_delta_scalar": 0.06464046239852905,
  "mean_abs_delta_scalar": 0.01434520035982132,
  "scalar_min": -14.578125,
  "scalar_max": 7.65234375,
  "scalar_absmax": 14.578125,
  "oracle_scalar_min": -14.593600273132324,
  "oracle_scalar_max": 7.658699989318848,
  "oracle_scalar_absmax": 14.593600273132324,
  "finite_all": true,
  "reset_identical": true,
  "session_resets_identical": true,
  "max_row_executions_per_process": 15,
  "provisional_max_delta_p_expectation": 0.02,
  "provisional_expectation_exceeded": false,
  "provisional_expectation_is_acceptance_gate": false,
  "worst_five_questions": [
   {
    "id": "request-11-q03",
    "request_id": "request-11",
    "type": "score",
    "zoo_only": false,
    "row_ids": [
     "request-11-q03-option-00",
     "request-11-q03-option-01",
     "request-11-q03-option-02"
    ],
    "options": [
     "none",
     "some",
     "complete"
    ],
    "nopts": 3,
    "scalars": [
     -1.9833984375,
     -0.7919921875,
     0.1658935546875
    ],
    "scalars_oracle": [
     -1.9961133003234863,
     -0.7945132851600647,
     0.13405287265777588
    ],
    "p": [
     0.1564840369177385,
     0.30912866310323367,
     0.5343872999790279
    ],
    "p_oracle": [
     0.15711289848380908,
     0.3121841461234784,
     0.5307029553927125
    ],
    "argmax": 2,
    "oracle_argmax": 2,
    "argmax_agrees": true,
    "selected_option": "complete",
    "oracle_selected_option": "complete",
    "oracle_margin": 0.21851880926923412,
    "abs_delta_p": [
     0.0006288615660705876,
     0.0030554830202447203,
     0.003684344586315391
    ],
    "max_abs_delta_p": 0.003684344586315391,
    "mean_abs_delta_p": 0.002456229724210233,
    "finite": true
   },
   {
    "id": "request-14-q04",
    "request_id": "request-14",
    "type": "noul",
    "zoo_only": false,
    "row_ids": [
     "request-14-q04-option-00",
     "request-14-q04-option-01"
    ],
    "options": [
     "yes",
     "no"
    ],
    "nopts": 2,
    "scalars": [
     -1.5673828125,
     1.375
    ],
    "scalars_oracle": [
     -1.5531220436096191,
     1.3508743047714233
    ],
    "p": [
     0.156915253827473,
     0.843084746172527
    ],
    "p_oracle": [
     0.15983899956016281,
     0.8401610004398371
    ],
    "argmax": 1,
    "oracle_argmax": 1,
    "argmax_agrees": true,
    "selected_option": "no",
    "oracle_selected_option": "no",
    "oracle_margin": 0.6803220008796743,
    "abs_delta_p": [
     0.002923745732689803,
     0.002923745732689942
    ],
    "max_abs_delta_p": 0.002923745732689942,
    "mean_abs_delta_p": 0.0029237457326898725,
    "finite": true
   },
   {
    "id": "request-15-q05",
    "request_id": "request-15",
    "type": "score",
    "zoo_only": false,
    "row_ids": [
     "request-15-q05-option-00",
     "request-15-q05-option-01",
     "request-15-q05-option-02",
     "request-15-q05-option-03",
     "request-15-q05-option-04"
    ],
    "options": [
     "very low",
     "low",
     "medium",
     "high",
     "critical"
    ],
    "nopts": 5,
    "scalars": [
     -6.4296875,
     -2.8125,
     -0.83984375,
     1.4873046875,
     -0.7861328125
    ],
    "scalars_oracle": [
     -6.44263219833374,
     -2.8297576904296875,
     -0.8664757013320923,
     1.476930022239685,
     -0.7765030860900879
    ],
    "p": [
     0.006638105123733906,
     0.052446354888491795,
     0.16190714281609525,
     0.6120549619234562,
     0.16695343524822281
    ],
    "p_oracle": [
     0.006626994094001005,
     0.052229686529919314,
     0.16037686679369648,
     0.61192850142172,
     0.16883795116066314
    ],
    "argmax": 3,
    "oracle_argmax": 3,
    "argmax_agrees": true,
    "selected_option": "high",
    "oracle_selected_option": "high",
    "oracle_margin": 0.4430905502610568,
    "abs_delta_p": [
     1.1111029732901317e-05,
     0.00021666835857248046,
     0.0015302760223987744,
     0.00012646050173625856,
     0.001884515912440321
    ],
    "max_abs_delta_p": 0.001884515912440321,
    "mean_abs_delta_p": 0.0007538063649761472,
    "finite": true
   },
   {
    "id": "request-20-q01",
    "request_id": "request-20",
    "type": "choice",
    "zoo_only": true,
    "row_ids": [
     "request-20-q01-option-00",
     "request-20-q01-option-01",
     "request-20-q01-option-02",
     "request-20-q01-option-03",
     "request-20-q01-option-04",
     "request-20-q01-option-05",
     "request-20-q01-option-06",
     "request-20-q01-option-07",
     "request-20-q01-option-08",
     "request-20-q01-option-09",
     "request-20-q01-option-10",
     "request-20-q01-option-11",
     "request-20-q01-option-12",
     "request-20-q01-option-13",
     "request-20-q01-option-14",
     "request-20-q01-option-15",
     "request-20-q01-option-16",
     "request-20-q01-option-17",
     "request-20-q01-option-18",
     "request-20-q01-option-19",
     "request-20-q01-option-20",
     "request-20-q01-option-21",
     "request-20-q01-option-22",
     "request-20-q01-option-23",
     "request-20-q01-option-24",
     "request-20-q01-option-25",
     "request-20-q01-option-26",
     "request-20-q01-option-27",
     "request-20-q01-option-28",
     "request-20-q01-option-29",
     "request-20-q01-option-30",
     "request-20-q01-option-31"
    ],
    "options": [
     "lighting",
     "plumbing",
     "heating",
     "cooling",
     "doors",
     "windows",
     "floors",
     "walls",
     "ceilings",
     "roofs",
     "stairs",
     "lifts",
     "locks",
     "shelves",
     "desks",
     "chairs",
     "curtains",
     "carpets",
     "drains",
     "pipes",
     "valves",
     "tanks",
     "pumps",
     "fans",
     "sockets",
     "switches",
     "cables",
     "sensors",
     "meters",
     "signs",
     "paths",
     "gates"
    ],
    "nopts": 32,
    "scalars": [
     -7.78515625,
     -7.390625,
     -8.3125,
     -8.359375,
     2.419921875,
     -5.3359375,
     -8.9140625,
     -6.8828125,
     -8.859375,
     -8.1484375,
     -9.203125,
     -6.06640625,
     -1.203125,
     -9.8359375,
     -10.5,
     -10.265625,
     -6.25,
     -10.7421875,
     -8.1015625,
     -7.09375,
     -2.53515625,
     -7.37890625,
     -6.9140625,
     -8.125,
     -7.984375,
     -4.9453125,
     -5.60546875,
     -3.224609375,
     -8.1953125,
     -9.203125,
     -7.89453125,
     4.1953125
    ],
    "scalars_oracle": [
     -7.80185079574585,
     -7.406404495239258,
     -8.328375816345215,
     -8.3722505569458,
     2.405498504638672,
     -5.341571807861328,
     -8.924745559692383,
     -6.8783721923828125,
     -8.876729011535645,
     -8.154253005981445,
     -9.206731796264648,
     -6.098209381103516,
     -1.189605712890625,
     -9.841769218444824,
     -10.507135391235352,
     -10.270621299743652,
     -6.233147144317627,
     -10.75239372253418,
     -8.108002662658691,
     -7.108526229858398,
     -2.5550146102905273,
     -7.36898946762085,
     -6.927596569061279,
     -8.134787559509277,
     -8.004584312438965,
     -4.9726715087890625,
     -5.6035261154174805,
     -3.2340099811553955,
     -8.21119499206543,
     -9.236793518066406,
     -7.896141052246094,
     4.17133903503418
    ],
    "p": [
     0.000717858944322546,
     0.0008993924878897198,
     0.000531091226422738,
     0.0005170544010148663,
     0.2446897370635693,
     0.0029097622762010423,
     0.0003766000720946237,
     0.0012021875005124253,
     0.0003885546416473148,
     0.0005832896021279521,
     0.0003192598143164164,
     0.0019168067904573286,
     0.0308667183897491,
     0.00022238232679585153,
     0.00015216018394047576,
     0.0001739664351356079,
     0.0017259022500446,
     0.00013249447553952243,
     0.0005991245593224508,
     0.0010656736134457185,
     0.014418582331831017,
     0.0009054354158788976,
     0.0011809104060285715,
     0.0005911540626877878,
     0.0006406183025131938,
     0.0036374601297118397,
     0.002494413243690419,
     0.009723489927841967,
     0.0005678731653653913,
     0.0003192598143164164,
     0.0006743660695540914,
     0.6748564200760307
    ],
    "p_oracle": [
     0.0007191815249930839,
     0.0009015207946759931,
     0.0005323186915273457,
     0.0005191386905016756,
     0.2454589060345332,
     0.0029336055303064035,
     0.0003785922027948519,
     0.0012190362342608175,
     0.00038912384915785244,
     0.0005880083232525815,
     0.0003222490363386637,
     0.0019038304734785964,
     0.03146211983276343,
     0.00022417928797102953,
     0.00015327548882480683,
     0.00017545591105207873,
     0.0017625481986839066,
     0.0001332316396700548,
     0.0006037558356560152,
     0.0010688079711317084,
     0.014419055372251894,
     0.0009210028446555952,
     0.001185224679289517,
     0.0005945853163533855,
     0.0006405108561773736,
     0.003622021856556868,
     0.002525765151669818,
     0.009782091018493599,
     0.0005691834701096167,
     0.0003167606763826054,
     0.0006814573372852584,
     0.6732934558692004
    ],
    "argmax": 31,
    "oracle_argmax": 31,
    "argmax_agrees": true,
    "selected_option": "gates",
    "oracle_selected_option": "gates",
    "oracle_margin": 0.42783454983466723,
    "abs_delta_p": [
     1.3225806705379102e-06,
     2.128306786273364e-06,
     1.2274651046077344e-06,
     2.084289486809377e-06,
     0.0007691689709638794,
     2.3843254105361273e-05,
     1.992130700228178e-06,
     1.6848733748392204e-05,
     5.692075105376304e-07,
     4.7187211246294295e-06,
     2.989222022247329e-06,
     1.2976316978732154e-05,
     0.0005954014430143324,
     1.7969611751780042e-06,
     1.115304884331071e-06,
     1.4894759164708362e-06,
     3.664594863930673e-05,
     7.371641305323835e-07,
     4.6312763335643785e-06,
     3.1343576859898907e-06,
     4.7304042087754905e-07,
     1.55674287766976e-05,
     4.3142732609453924e-06,
     3.4312536655976703e-06,
     1.0744633582014142e-07,
     1.543827315497173e-05,
     3.135190797939891e-05,
     5.860109065163217e-05,
     1.3103047442254321e-06,
     2.499137933811011e-06,
     7.091267731167074e-06,
     0.0015629642068303307
    ],
    "max_abs_delta_p": 0.0015629642068303307,
    "mean_abs_delta_p": 9.962408632710677e-05,
    "finite": true
   },
   {
    "id": "request-13-q05",
    "request_id": "request-13",
    "type": "score",
    "zoo_only": false,
    "row_ids": [
     "request-13-q05-option-00",
     "request-13-q05-option-01",
     "request-13-q05-option-02",
     "request-13-q05-option-03"
    ],
    "options": [
     "unusable",
     "limited",
     "mostly usable",
     "fully usable"
    ],
    "nopts": 4,
    "scalars": [
     1.6240234375,
     0.2861328125,
     -1.291015625,
     -3.94921875
    ],
    "scalars_oracle": [
     1.6196714639663696,
     0.27900874614715576,
     -1.3120677471160889,
     -3.97291898727417
    ],
    "p": [
     0.5896212147307447,
     0.27450552636424547,
     0.11146868669589143,
     0.024404572209118385
    ],
    "p_oracle": [
     0.5906616660481768,
     0.274554667682575,
     0.11060483754235892,
     0.024178828726889312
    ],
    "argmax": 0,
    "oracle_argmax": 0,
    "argmax_agrees": true,
    "selected_option": "unusable",
    "oracle_selected_option": "unusable",
    "oracle_margin": 0.3161069983656018,
    "abs_delta_p": [
     0.001040451317432134,
     4.91413183295597e-05,
     0.000863849153532506,
     0.00022574348222907312
    ],
    "max_abs_delta_p": 0.001040451317432134,
    "mean_abs_delta_p": 0.0005447963178808182,
    "finite": true
   }
  ]
 },
 "reset_check": {
  "row": "request-01-q01-option-00",
  "identical": true,
  "comparison": "dtype and exact bytes",
  "abs_diff": 0.0,
  "first_scalar": 4.3125,
  "final_scalar": 4.3125,
  "first_bits": "5044",
  "final_bits": "5044",
  "final_session": "fp16-020-global_reset-a1",
  "cross_process": true,
  "local_resets_in_every_session": true
 },
 "wall_seconds_contended": 725.1608537080465,
 "completed_at": "2026-09-23T00:54:10+00:00"
}