{ "schema": "coreai-decider-fixtures/1", "source": { "hf_id": "Mapika/decider-0.8b", "revision": "1ea54127d3bd52f6d753d9257b32a6380b873907", "oracle": "author's decider/model.py slot_logits, fp32, CPU", "torch": "2.9.0" }, "temperature": 1.03, "layout": "state_first", "independent_rows": true, "isolated_levels": true, "neutralize_none": false, "label_table": [ [ "A", 32 ], [ "B", 33 ], [ "C", 34 ], [ "D", 35 ], [ "E", 36 ], [ "F", 37 ], [ "G", 38 ], [ "H", 39 ], [ "I", 40 ], [ "J", 41 ], [ "K", 42 ], [ "L", 43 ], [ "M", 44 ], [ "N", 45 ], [ "O", 46 ], [ "P", 47 ], [ "Q", 48 ], [ "R", 49 ], [ "S", 50 ], [ "T", 51 ], [ "U", 52 ], [ "V", 53 ], [ "W", 54 ], [ "X", 55 ], [ "Y", 56 ], [ "Z", 57 ], [ "AA", 5840 ], [ "AB", 1803 ], [ "AC", 1646 ], [ "AD", 1745 ], [ "AE", 13276 ], [ "AF", 8018 ], [ "AG", 1825 ], [ "AH", 28946 ], [ "AI", 15015 ], [ "AJ", 29595 ], [ "AK", 11568 ], [ "AL", 939 ], [ "AM", 1354 ], [ "AN", 1058 ], [ "AO", 18183 ], [ "AP", 2456 ], [ "AQ", 88898 ], [ "AR", 905 ], [ "AS", 1846 ], [ "AT", 802 ], [ "AU", 33869 ], [ "AV", 7839 ], [ "AW", 14006 ], [ "AX", 2860 ], [ "AY", 2926 ], [ "AZ", 22828 ], [ "BA", 6844 ], [ "BB", 9798 ], [ "BC", 4738 ], [ "BD", 9265 ], [ "BE", 11261 ], [ "BF", 19278 ], [ "BG", 36513 ], [ "BH", 93801 ], [ "BI", 8335 ], [ "BJ", 14544 ], [ "BK", 85266 ], [ "BL", 9110 ], [ "BM", 28000 ], [ "BN", 15137 ], [ "BO", 4525 ], [ "BP", 25261 ], [ "BR", 12717 ], [ "BS", 7116 ], [ "BT", 17078 ], [ "BU", 14497 ], [ "BV", 57339 ], [ "BW", 74909 ], [ "BX", 52072 ], [ "BY", 19305 ], [ "CA", 4887 ], [ "CB", 12607 ], [ "CC", 3580 ], [ "CD", 6281 ], [ "CE", 2036 ], [ "CF", 9362 ], [ "CG", 8533 ], [ "CH", 2080 ], [ "CI", 10911 ], [ "CK", 2925 ], [ "CL", 3040 ], [ "CM", 9690 ], [ "CN", 27731 ], [ "CO", 8023 ], [ "CP", 6901 ], [ "CR", 8702 ], [ "CS", 6211 ], [ "CT", 1123 ], [ "CU", 16307 ], [ "CV", 18990 ], [ "CW", 64045 ], [ "CX", 63037 ], [ "CY", 33380 ], [ "DA", 6151 ], [ "DB", 3392 ], [ "DC", 5449 ], [ "DD", 3967 ], [ "DE", 1113 ], [ "DF", 5095 ], [ "DG", 51923 ], [ "DH", 49600 ], [ "DI", 17099 ], [ "DJ", 51483 ], [ "DK", 17756 ], [ "DL", 16037 ], [ "DM", 8135 ], [ "DN", 30237 ], [ "DO", 5683 ], [ "DP", 9992 ], [ "DR", 7444 ], [ "DS", 5751 ], [ "DT", 10284 ], [ "DU", 20887 ], [ "DV", 59884 ], [ "DW", 52396 ], [ "DX", 16103 ], [ "DY", 67547 ], [ "EA", 18535 ], [ "EB", 8006 ], [ "EC", 7263 ], [ "ED", 1425 ], [ "EE", 6878 ], [ "EF", 14453 ], [ "EG", 9097 ], [ "EH", 44072 ], [ "EI", 76089 ], [ "EK", 68720 ], [ "EL", 2662 ], [ "EM", 2629 ], [ "EN", 923 ], [ "EO", 6548 ], [ "EP", 8924 ], [ "EQ", 52194 ], [ "ER", 622 ], [ "ES", 1515 ], [ "ET", 1300 ], [ "EU", 37523 ], [ "EV", 44473 ], [ "EW", 36530 ], [ "EX", 3152 ], [ "EZ", 93924 ], [ "FA", 3505 ], [ "FB", 15731 ], [ "FC", 6542 ], [ "FD", 14176 ], [ "FE", 11091 ], [ "FF", 1686 ], [ "FG", 11660 ], [ "FH", 80440 ], [ "FI", 18836 ], [ "FK", 26132 ], [ "FL", 5934 ], [ "FM", 24794 ], [ "FN", 40229 ], [ "FO", 3660 ], [ "FP", 11361 ], [ "FR", 10191 ], [ "FS", 8225 ], [ "FT", 3860 ], [ "FU", 78413 ], [ "FW", 17680 ], [ "FX", 15900 ], [ "FY", 78138 ], [ "GA", 15653 ], [ "GB", 5213 ], [ "GC", 22150 ], [ "GD", 39500 ], [ "GE", 10460 ], [ "GF", 35131 ], [ "GG", 21563 ], [ "GH", 43194 ], [ "GI", 27127 ], [ "GL", 3697 ], [ "GM", 20011 ], [ "GN", 24368 ], [ "GO", 15058 ], [ "GP", 23658 ], [ "GR", 8362 ], [ "GS", 16035 ], [ "GT", 24583 ], [ "GU", 52857 ], [ "GV", 38403 ], [ "GW", 60456 ], [ "GX", 81519 ], [ "GY", 40097 ], [ "HA", 16522 ], [ "HB", 29722 ], [ "HC", 21756 ], [ "HD", 18567 ], [ "HE", 1736 ], [ "HF", 48043 ], [ "HG", 87013 ], [ "HH", 22456 ], [ "HI", 23165 ], [ "HK", 55523 ], [ "HL", 13097 ], [ "HM", 50397 ], [ "HN", 41741 ], [ "HO", 23073 ], [ "HP", 6401 ], [ "HQ", 86538 ], [ "HR", 16585 ], [ "HS", 11622 ], [ "HT", 2464 ], [ "HU", 84982 ], [ "HV", 75516 ], [ "HW", 36984 ], [ "HX", 58795 ], [ "HY", 47217 ], [ "HZ", 59675 ], [ "IA", 5681 ], [ "IB", 3151 ], [ "IC", 1271 ], [ "ID", 887 ], [ "IE", 5203 ], [ "IF", 2685 ], [ "IG", 1849 ], [ "IH", 72470 ], [ "II", 5370 ], [ "IJ", 74063 ], [ "IK", 27629 ], [ "IL", 1655 ], [ "IM", 1728 ], [ "IN", 669 ], [ "IO", 3682 ], [ "IP", 3191 ], [ "IQ", 59865 ], [ "IR", 2712 ], [ "IS", 1580 ], [ "IT", 922 ], [ "IU", 77243 ], [ "IV", 2990 ], [ "IW", 78493 ], [ "IX", 5228 ], [ "IZ", 2750 ], [ "JA", 42711 ], [ "JB", 44568 ], [ "JC", 56402 ], [ "JD", 48236 ], [ "JE", 38993 ], [ "JI", 43559 ], [ "JJ", 61250 ], [ "JK", 32942 ], [ "JM", 86302 ], [ "JO", 25310 ], [ "JP", 26313 ], [ "JR", 81770 ], [ "JS", 12185 ], [ "JT", 78382 ] ], "requests": [ { "id": "r01", "state": "A parcel has a broken latch. Its contents are intact. The sender asks for a replacement latch, not a refund.", "questions": { "route": { "type": "choice", "instructions": "Which team should handle the request?", "criteria": { "repair": "Replace damaged parts", "billing": "Handle payments", "delivery": "Find missing parcels" } }, "condition": { "type": "choice", "instructions": "What is the condition of the contents?", "criteria": { "missing": null, "intact": null, "wet": null, "broken": null } }, "refund": { "type": "noul", "instructions": "Does the sender request a refund?", "criteria": { "false": "No money back is requested", "true": "Money back is requested" } } } }, { "id": "r02", "state": { "room": { "temperature": "cold", "lamp": "off", "window": "open" }, "request": "Close the window." }, "questions": { "action": { "type": "choice", "instructions": "What action is requested?", "criteria": { "open the window": null, "close the window": null, "turn on the lamp": null } }, "lamp": { "type": "choice", "instructions": "What is the current lamp state?", "criteria": { "on": null, "flashing": null, "off": null } }, "cold": { "type": "noul", "instructions": "Is the room described as cold?", "criteria": {} } } }, { "id": "r03", "state": "The marked tile is a green hexagon. It is dry and has no border.", "questions": { "color": { "type": "choice", "instructions": "What color is the marked tile?", "criteria": { "red": null, "blue": null, "yellow": null, "purple": null, "green": null, "orange": null } }, "shape": { "type": "choice", "instructions": "What shape is the marked tile?", "criteria": { "circle": null, "square": null, "triangle": null, "pentagon": null, "hexagon": null, "oval": null, "star": null, "rectangle": null } }, "wet": { "type": "noul", "instructions": "Is the marked tile wet?", "criteria": { "false": "The tile is dry", "true": "The tile has water on it" } } } }, { "id": "r04", "state": { "cabinet": { "drawer": "seven", "material": "wood", "status": "locked" }, "key": "present" }, "questions": { "drawer": { "type": "choice", "instructions": "Which drawer is named?", "criteria": { "one": null, "two": null, "three": null, "four": null, "five": null, "six": null, "seven": null } }, "material": { "type": "choice", "instructions": "What material is the cabinet made from?", "criteria": { "glass": null, "stone": null, "steel": null, "paper": null, "clay": null, "wood": null, "rubber": null, "copper": null, "fabric": null, "plastic": null } }, "key": { "type": "noul", "instructions": "Is the key present?", "criteria": {} } } }, { "id": "r05", "state": "A garden bed is dry. The seedlings have green leaves. The next task is watering.", "questions": { "task": { "type": "choice", "instructions": "What is the next task?", "criteria": { "harvesting": null, "watering": null, "painting": null } }, "leaves": { "type": "choice", "instructions": "What color are the leaves?", "criteria": { "brown": null, "yellow": null, "green": null, "red": null } }, "dry": { "type": "noul", "instructions": "Is the garden bed dry?", "criteria": {} } } }, { "id": "r06", "state": { "job": { "phase": "review", "result": "pending", "priority": "low" }, "approved": false }, "questions": { "phase": { "type": "choice", "instructions": "Which phase is the job in?", "criteria": { "draft": null, "review": null, "complete": null, "cancelled": null } }, "priority": { "type": "choice", "instructions": "What is the job priority?", "criteria": { "high": null, "medium": null, "low": null } }, "approved": { "type": "noul", "instructions": "Has the job been approved?", "criteria": { "false": "Approval has not been granted", "true": "Approval has been granted" } } } }, { "id": "r07", "state": "A container holds sand. Its lid is blue. A label says to keep it indoors, and the seal is unbroken.", "questions": { "contents": { "type": "choice", "instructions": "What does the container hold?", "criteria": { "water": null, "sand": null, "salt": null, "seeds": null } }, "location": { "type": "choice", "instructions": "Where should the container be kept?", "criteria": { "outdoors": null, "underground": null, "indoors": null, "on a roof": null, "in a pond": null } }, "seal": { "type": "noul", "instructions": "Is the seal broken?", "criteria": {} } } }, { "id": "r08", "state": { "route": { "direction": "west", "surface": "gravel", "closed": true }, "weather": "clear" }, "questions": { "direction": { "type": "choice", "instructions": "Which direction does the route lead?", "criteria": { "north": null, "south": null, "east": null, "west": null, "up": null } }, "surface": { "type": "choice", "instructions": "What is the route surface?", "criteria": { "gravel": null, "ice": null, "asphalt": null } }, "open": { "type": "noul", "instructions": "Is the route open?", "criteria": { "false": "The route is closed", "true": "Travel is permitted" } } } }, { "id": "r09", "state": "The storage tank is half full of clean water. The valve is closed. Its fill level is exactly halfway between empty and full.", "questions": { "liquid": { "type": "choice", "instructions": "What liquid is in the tank?", "criteria": { "oil": null, "water": null, "ink": null } }, "valve": { "type": "choice", "instructions": "What is the valve position?", "criteria": { "open": null, "closed": null, "missing": null } }, "clarity": { "type": "choice", "instructions": "How is the water described?", "criteria": { "dirty": null, "cloudy": null, "frozen": null, "clean": null } }, "fill": { "type": "score", "instructions": "How full is the tank?", "criteria": [ "empty", "one quarter full", "half full", "three quarters full", "completely full" ] } } }, { "id": "r10", "state": { "inspection": { "damage": "none", "surface": "smooth", "color": "white", "count": 4 } }, "questions": { "surface": { "type": "choice", "instructions": "How is the inspected surface described?", "criteria": { "rough": null, "smooth": null, "cracked": null, "sticky": null } }, "color": { "type": "choice", "instructions": "What is the surface color?", "criteria": { "black": null, "white": null, "gray": null } }, "count": { "type": "choice", "instructions": "How many items were inspected?", "criteria": { "two": null, "four": null, "six": null } }, "damage": { "type": "score", "instructions": "How much damage is present?", "criteria": [ "no damage", "minor scratches", "several shallow dents", "large cracks", "completely broken" ] } } }, { "id": "r11", "state": "The selected storage bin is violet. Its lid is closed.", "questions": { "bin": { "type": "choice", "instructions": "Which storage bin is selected?", "criteria": { "red": null, "orange": null, "yellow": null, "green": null, "blue": null, "indigo": null, "violet": null, "black": null, "white": null, "gray": null, "brown": null } } } }, { "id": "r12", "state": "The requested archive slot is 173. Retrieve that slot.", "questions": { "slot": { "type": "choice", "instructions": "Which archive slot is requested?", "criteria": { "1": null, "2": null, "3": null, "4": null, "5": null, "6": null, "7": null, "8": null, "9": null, "10": null, "11": null, "12": null, "13": null, "14": null, "15": null, "16": null, "17": null, "18": null, "19": null, "20": null, "21": null, "22": null, "23": null, "24": null, "25": null, "26": null, "27": null, "28": null, "29": null, "30": null, "31": null, "32": null, "33": null, "34": null, "35": null, "36": null, "37": null, "38": null, "39": null, "40": null, "41": null, "42": null, "43": null, "44": null, "45": null, "46": null, "47": null, "48": null, "49": null, "50": null, "51": null, "52": null, "53": null, "54": null, "55": null, "56": null, "57": null, "58": null, "59": null, "60": null, "61": null, "62": null, "63": null, "64": null, "65": null, "66": null, "67": null, "68": null, "69": null, "70": null, "71": null, "72": null, "73": null, "74": null, "75": null, "76": null, "77": null, "78": null, "79": null, "80": null, "81": null, "82": null, "83": null, "84": null, "85": null, "86": null, "87": null, "88": null, "89": null, "90": null, "91": null, "92": null, "93": null, "94": null, "95": null, "96": null, "97": null, "98": null, "99": null, "100": null, "101": null, "102": null, "103": null, "104": null, "105": null, "106": null, "107": null, "108": null, "109": null, "110": null, "111": null, "112": null, "113": null, "114": null, "115": null, "116": null, "117": null, "118": null, "119": null, "120": null, "121": null, "122": null, "123": null, "124": null, "125": null, "126": null, "127": null, "128": null, "129": null, "130": null, "131": null, "132": null, "133": null, "134": null, "135": null, "136": null, "137": null, "138": null, "139": null, "140": null, "141": null, "142": null, "143": null, "144": null, "145": null, "146": null, "147": null, "148": null, "149": null, "150": null, "151": null, "152": null, "153": null, "154": null, "155": null, "156": null, "157": null, "158": null, "159": null, "160": null, "161": null, "162": null, "163": null, "164": null, "165": null, "166": null, "167": null, "168": null, "169": null, "170": null, "171": null, "172": null, "173": null, "174": null, "175": null, "176": null, "177": null, "178": null, "179": null, "180": null, "181": null, "182": null, "183": null, "184": null, "185": null, "186": null, "187": null, "188": null, "189": null, "190": null, "191": null, "192": null, "193": null, "194": null, "195": null, "196": null, "197": null, "198": null, "199": null, "200": null, "201": null, "202": null, "203": null, "204": null, "205": null, "206": null, "207": null, "208": null, "209": null, "210": null, "211": null, "212": null, "213": null, "214": null, "215": null, "216": null, "217": null, "218": null, "219": null, "220": null, "221": null, "222": null, "223": null, "224": null, "225": null, "226": null, "227": null, "228": null, "229": null, "230": null, "231": null, "232": null, "233": null, "234": null, "235": null, "236": null, "237": null, "238": null, "239": null, "240": null, "241": null, "242": null, "243": null, "244": null, "245": null, "246": null, "247": null, "248": null, "249": null, "250": null, "251": null, "252": null, "253": null, "254": null, "255": null } } } }, { "id": "r13", "state": { "notice": { "status": "paused", "reason": "inspection" }, "restart_allowed": false }, "questions": { "status": { "type": "choice", "instructions": "What is the current status?", "criteria": { "running": null, "paused": null, "finished": null } }, "restart": { "type": "noul", "instructions": "Is a restart currently allowed?", "criteria": {} } } } ], "rows": [ { "id": "r01-route", "request_id": "r01", "question_id": "route", "kind": "list", "type": "choice", "level_index": null, "question": "Which team should handle the request?", "options": [ "repair: Replace damaged parts", "billing: Handle payments", "delivery: Find missing parcels" ], "ids": [ 1905, 25, 198, 32, 28350, 682, 264, 10548, 55022, 13, 11116, 8529, 513, 33295, 13, 561, 4495, 16561, 364, 264, 13329, 55022, 11, 524, 264, 20325, 13, 271, 14162, 25, 15451, 2015, 1220, 3579, 279, 1622, 30, 198, 3670, 25, 198, 4205, 8, 12368, 25, 28592, 19510, 5306, 198, 5181, 8, 32419, 25, 13365, 13767, 198, 2929, 8, 9403, 25, 7145, 7167, 81031, 198, 15666, 25, 318 ], "slot": 66, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 67, "slot_logits": [ 15.594087600708008, 11.969778060913086, 7.974287033081055 ], "p_oracle": [ 0.9706390500068665, 0.02876637876033783, 0.0005945737357251346 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9418726563453674 }, { "id": "r01-condition", "request_id": "r01", "question_id": "condition", "kind": "list", "type": "choice", "level_index": null, "question": "What is the condition of the contents?", "options": [ "missing", "intact", "wet", "broken" ], "ids": [ 1905, 25, 198, 32, 28350, 682, 264, 10548, 55022, 13, 11116, 8529, 513, 33295, 13, 561, 4495, 16561, 364, 264, 13329, 55022, 11, 524, 264, 20325, 13, 271, 14162, 25, 3437, 369, 279, 2876, 314, 279, 8529, 30, 198, 3670, 25, 198, 4205, 8, 7167, 198, 5181, 8, 33295, 198, 2929, 8, 13988, 198, 5263, 8, 10548, 198, 15666, 25, 318 ], "slot": 60, "nopts": 4, "label_ids": [ 32, 33, 34, 35 ], "label_strings": [ "A", "B", "C", "D" ], "tokens": 61, "slot_logits": [ 9.695514678955078, 16.17848777770996, 8.77947998046875, 10.273113250732422 ], "p_oracle": [ 0.0018363530980423093, 0.9941917061805725, 0.0007545964326709509, 0.0032173478975892067 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9909743666648865 }, { "id": "r01-refund", "request_id": "r01", "question_id": "refund", "kind": "list", "type": "noul", "level_index": null, "question": "Does the sender request a refund?", "options": [ "no: No money back is requested", "yes: Money back is requested" ], "ids": [ 1905, 25, 198, 32, 28350, 682, 264, 10548, 55022, 13, 11116, 8529, 513, 33295, 13, 561, 4495, 16561, 364, 264, 13329, 55022, 11, 524, 264, 20325, 13, 271, 14162, 25, 12193, 279, 4495, 1622, 264, 20325, 30, 198, 3670, 25, 198, 4205, 8, 874, 25, 2233, 3117, 1142, 369, 10897, 198, 5181, 8, 9542, 25, 17107, 1142, 369, 10897, 198, 15666, 25, 318 ], "slot": 62, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 63, "slot_logits": [ 15.701435089111328, 9.583717346191406 ], "p_oracle": [ 0.9973737001419067, 0.0026263452600687742 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9947473406791687 }, { "id": "r02-action", "request_id": "r02", "question_id": "action", "kind": "list", "type": "choice", "level_index": null, "question": "What action is requested?", "options": [ "open the window", "close the window", "turn on the lamp" ], "ids": [ 1905, 25, 198, 4754, 2871, 763, 5046, 33413, 763, 328, 84068, 487, 328, 91485, 763, 328, 1783, 487, 328, 5334, 763, 328, 2428, 13933, 328, 1967, 763, 328, 7675, 279, 3136, 1149, 92, 271, 14162, 25, 3437, 1852, 369, 10897, 30, 198, 3670, 25, 198, 4205, 8, 1724, 279, 3136, 198, 5181, 8, 3160, 279, 3136, 198, 2929, 8, 2404, 383, 279, 27056, 198, 15666, 25, 318 ], "slot": 66, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 67, "slot_logits": [ 10.437089920043945, 17.73750877380371, 7.838837146759033 ], "p_oracle": [ 0.0008344926754944026, 0.999098539352417, 6.697019125567749e-05 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9982640743255615 }, { "id": "r02-lamp", "request_id": "r02", "question_id": "lamp", "kind": "list", "type": "choice", "level_index": null, "question": "What is the current lamp state?", "options": [ "on", "flashing", "off" ], "ids": [ 1905, 25, 198, 4754, 2871, 763, 5046, 33413, 763, 328, 84068, 487, 328, 91485, 763, 328, 1783, 487, 328, 5334, 763, 328, 2428, 13933, 328, 1967, 763, 328, 7675, 279, 3136, 1149, 92, 271, 14162, 25, 3437, 369, 279, 1428, 27056, 1528, 30, 198, 3670, 25, 198, 4205, 8, 383, 198, 5181, 8, 48084, 198, 2929, 8, 974, 198, 15666, 25, 318 ], "slot": 61, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 62, "slot_logits": [ 12.740621566772461, 8.857020378112793, 14.897481918334961 ], "p_oracle": [ 0.10940099507570267, 0.002520693466067314, 0.8880782723426819 ], "argmax": 2, "argmax_label": "C", "argmax_token_id": 34, "top2_margin": 0.7786772847175598 }, { "id": "r02-cold", "request_id": "r02", "question_id": "cold", "kind": "list", "type": "noul", "level_index": null, "question": "Is the room described as cold?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 4754, 2871, 763, 5046, 33413, 763, 328, 84068, 487, 328, 91485, 763, 328, 1783, 487, 328, 5334, 763, 328, 2428, 13933, 328, 1967, 763, 328, 7675, 279, 3136, 1149, 92, 271, 14162, 25, 2091, 279, 2956, 7246, 430, 8981, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 57, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 58, "slot_logits": [ 10.351333618164062, 13.59166431427002 ], "p_oracle": [ 0.04125106334686279, 0.9587488770484924 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9174978137016296 }, { "id": "r03-color", "request_id": "r03", "question_id": "color", "kind": "list", "type": "choice", "level_index": null, "question": "What color is the marked tile?", "options": [ "red", "blue", "yellow", "purple", "green", "orange" ], "ids": [ 1905, 25, 198, 760, 12495, 9727, 369, 264, 5983, 12017, 5922, 13, 1049, 369, 8785, 321, 682, 874, 3755, 13, 271, 14162, 25, 3437, 1829, 369, 279, 12495, 9727, 30, 198, 3670, 25, 198, 4205, 8, 2438, 198, 5181, 8, 6105, 198, 2929, 8, 13358, 198, 5263, 8, 24142, 198, 10406, 8, 5983, 198, 7584, 8, 18016, 198, 15666, 25, 318 ], "slot": 60, "nopts": 6, "label_ids": [ 32, 33, 34, 35, 36, 37 ], "label_strings": [ "A", "B", "C", "D", "E", "F" ], "tokens": 61, "slot_logits": [ 10.76678466796875, 10.290726661682129, 11.609323501586914, 10.098481178283691, 15.342511177062988, 10.711405754089355 ], "p_oracle": [ 0.011068147607147694, 0.006971844006329775, 0.025079987943172455, 0.005784797016531229, 0.9406063556671143, 0.010488779284060001 ], "argmax": 4, "argmax_label": "E", "argmax_token_id": 36, "top2_margin": 0.9155263900756836 }, { "id": "r03-shape", "request_id": "r03", "question_id": "shape", "kind": "list", "type": "choice", "level_index": null, "question": "What shape is the marked tile?", "options": [ "circle", "square", "triangle", "pentagon", "hexagon", "oval", "star", "rectangle" ], "ids": [ 1905, 25, 198, 760, 12495, 9727, 369, 264, 5983, 12017, 5922, 13, 1049, 369, 8785, 321, 682, 874, 3755, 13, 271, 14162, 25, 3437, 5893, 369, 279, 12495, 9727, 30, 198, 3670, 25, 198, 4205, 8, 12307, 198, 5181, 8, 9057, 198, 2929, 8, 20837, 198, 5263, 8, 19576, 5922, 198, 10406, 8, 12017, 5922, 198, 7584, 8, 58011, 198, 6489, 8, 6561, 198, 10578, 8, 22048, 198, 15666, 25, 318 ], "slot": 70, "nopts": 8, "label_ids": [ 32, 33, 34, 35, 36, 37, 38, 39 ], "label_strings": [ "A", "B", "C", "D", "E", "F", "G", "H" ], "tokens": 71, "slot_logits": [ 10.392077445983887, 10.90335750579834, 11.135916709899902, 11.309795379638672, 16.040969848632812, 9.915843963623047, 10.67845344543457, 10.65204906463623 ], "p_oracle": [ 0.003979754168540239, 0.006537850946187973, 0.008193938992917538, 0.00970080029219389, 0.9587033987045288, 0.0025064260698854923, 0.005255395546555519, 0.005122386384755373 ], "argmax": 4, "argmax_label": "E", "argmax_token_id": 36, "top2_margin": 0.9490026235580444 }, { "id": "r03-wet", "request_id": "r03", "question_id": "wet", "kind": "list", "type": "noul", "level_index": null, "question": "Is the marked tile wet?", "options": [ "no: The tile is dry", "yes: The tile has water on it" ], "ids": [ 1905, 25, 198, 760, 12495, 9727, 369, 264, 5983, 12017, 5922, 13, 1049, 369, 8785, 321, 682, 874, 3755, 13, 271, 14162, 25, 2091, 279, 12495, 9727, 13988, 30, 198, 3670, 25, 198, 4205, 8, 874, 25, 561, 9727, 369, 8785, 198, 5181, 8, 9542, 25, 561, 9727, 682, 2919, 383, 424, 198, 15666, 25, 318 ], "slot": 55, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 56, "slot_logits": [ 15.09995174407959, 9.292283058166504 ], "p_oracle": [ 0.9964544773101807, 0.003545538056641817 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9929089546203613 }, { "id": "r04-drawer", "request_id": "r04", "question_id": "drawer", "kind": "list", "type": "choice", "level_index": null, "question": "Which drawer is named?", "options": [ "one", "two", "three", "four", "five", "six", "seven" ], "ids": [ 1905, 25, 198, 4754, 66, 72935, 763, 5046, 77364, 763, 328, 77420, 487, 328, 7986, 763, 328, 6451, 487, 328, 2738, 763, 328, 23857, 13933, 328, 767, 763, 328, 27803, 8934, 271, 14162, 25, 15451, 25635, 369, 6725, 30, 198, 3670, 25, 198, 4205, 8, 799, 198, 5181, 8, 1330, 198, 2929, 8, 2250, 198, 5263, 8, 2943, 198, 10406, 8, 4097, 198, 7584, 8, 4590, 198, 6489, 8, 7840, 198, 15666, 25, 318 ], "slot": 73, "nopts": 7, "label_ids": [ 32, 33, 34, 35, 36, 37, 38 ], "label_strings": [ "A", "B", "C", "D", "E", "F", "G" ], "tokens": 74, "slot_logits": [ 11.870087623596191, 10.166424751281738, 9.513827323913574, 9.715113639831543, 9.895279884338379, 9.523934364318848, 16.415470123291016 ], "p_oracle": [ 0.011879051104187965, 0.00227216980420053, 0.0012058026622980833, 0.0014660466695204377, 0.001746279769577086, 0.0012176926247775555, 0.9802129864692688 ], "argmax": 6, "argmax_label": "G", "argmax_token_id": 38, "top2_margin": 0.9683339595794678 }, { "id": "r04-material", "request_id": "r04", "question_id": "material", "kind": "list", "type": "choice", "level_index": null, "question": "What material is the cabinet made from?", "options": [ "glass", "stone", "steel", "paper", "clay", "wood", "rubber", "copper", "fabric", "plastic" ], "ids": [ 1905, 25, 198, 4754, 66, 72935, 763, 5046, 77364, 763, 328, 77420, 487, 328, 7986, 763, 328, 6451, 487, 328, 2738, 763, 328, 23857, 13933, 328, 767, 763, 328, 27803, 8934, 271, 14162, 25, 3437, 3558, 369, 279, 21245, 1801, 494, 30, 198, 3670, 25, 198, 4205, 8, 8720, 198, 5181, 8, 9506, 198, 2929, 8, 9229, 198, 5263, 8, 5392, 198, 10406, 8, 34854, 198, 7584, 8, 7339, 198, 6489, 8, 21970, 198, 10578, 8, 22561, 198, 8701, 8, 12681, 198, 15887, 8, 11840, 198, 15666, 25, 318 ], "slot": 88, "nopts": 10, "label_ids": [ 32, 33, 34, 35, 36, 37, 38, 39, 40, 41 ], "label_strings": [ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" ], "tokens": 89, "slot_logits": [ 9.451681137084961, 10.713202476501465, 12.030733108520508, 9.702373504638672, 9.380693435668945, 15.686629295349121, 9.584344863891602, 10.500152587890625, 9.01577091217041, 10.532502174377441 ], "p_oracle": [ 0.002213399624451995, 0.007533108815550804, 0.027071012184023857, 0.002823340008035302, 0.002065989887341857, 0.9418793320655823, 0.002517657820135355, 0.006125518586486578, 0.0014496404910460114, 0.006320958025753498 ], "argmax": 5, "argmax_label": "F", "argmax_token_id": 37, "top2_margin": 0.9148083329200745 }, { "id": "r04-key", "request_id": "r04", "question_id": "key", "kind": "list", "type": "noul", "level_index": null, "question": "Is the key present?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 4754, 66, 72935, 763, 5046, 77364, 763, 328, 77420, 487, 328, 7986, 763, 328, 6451, 487, 328, 2738, 763, 328, 23857, 13933, 328, 767, 763, 328, 27803, 8934, 271, 14162, 25, 2091, 279, 1328, 2945, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 53, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 54, "slot_logits": [ 10.400930404663086, 14.606792449951172 ], "p_oracle": [ 0.01657148450613022, 0.9834285378456116 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9668570756912231 }, { "id": "r05-task", "request_id": "r05", "question_id": "task", "kind": "list", "type": "choice", "level_index": null, "question": "What is the next task?", "options": [ "harvesting", "watering", "painting" ], "ids": [ 1905, 25, 198, 32, 13159, 4690, 369, 8785, 13, 561, 10014, 18243, 599, 5983, 10583, 13, 561, 1727, 3274, 369, 70269, 13, 271, 14162, 25, 3437, 369, 279, 1727, 3274, 30, 198, 3670, 25, 198, 4205, 8, 63433, 198, 5181, 8, 70269, 198, 2929, 8, 18255, 198, 15666, 25, 318 ], "slot": 49, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 50, "slot_logits": [ 10.84885311126709, 16.44983673095703, 8.74946403503418 ], "p_oracle": [ 0.004327560309320688, 0.9951086640357971, 0.0005636969581246376 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9907811284065247 }, { "id": "r05-leaves", "request_id": "r05", "question_id": "leaves", "kind": "list", "type": "choice", "level_index": null, "question": "What color are the leaves?", "options": [ "brown", "yellow", "green", "red" ], "ids": [ 1905, 25, 198, 32, 13159, 4690, 369, 8785, 13, 561, 10014, 18243, 599, 5983, 10583, 13, 561, 1727, 3274, 369, 70269, 13, 271, 14162, 25, 3437, 1829, 513, 279, 10583, 30, 198, 3670, 25, 198, 4205, 8, 13477, 198, 5181, 8, 13358, 198, 2929, 8, 5983, 198, 5263, 8, 2438, 198, 15666, 25, 318 ], "slot": 53, "nopts": 4, "label_ids": [ 32, 33, 34, 35 ], "label_strings": [ "A", "B", "C", "D" ], "tokens": 54, "slot_logits": [ 8.635403633117676, 10.057342529296875, 16.268339157104492, 9.046842575073242 ], "p_oracle": [ 0.0006024409085512161, 0.0023958953097462654, 0.9961035251617432, 0.0008982465951703489 ], "argmax": 2, "argmax_label": "C", "argmax_token_id": 34, "top2_margin": 0.9937076568603516 }, { "id": "r05-dry", "request_id": "r05", "question_id": "dry", "kind": "list", "type": "noul", "level_index": null, "question": "Is the garden bed dry?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 32, 13159, 4690, 369, 8785, 13, 561, 10014, 18243, 599, 5983, 10583, 13, 561, 1727, 3274, 369, 70269, 13, 271, 14162, 25, 2091, 279, 13159, 4690, 8785, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 45, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 46, "slot_logits": [ 10.509957313537598, 14.470795631408691 ], "p_oracle": [ 0.02092888019979, 0.9790711402893066 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9581422805786133 }, { "id": "r06-phase", "request_id": "r06", "question_id": "phase", "kind": "list", "type": "choice", "level_index": null, "question": "Which phase is the job in?", "options": [ "draft", "review", "complete", "cancelled" ], "ids": [ 1905, 25, 198, 4754, 8534, 763, 5046, 24344, 763, 328, 18828, 487, 328, 1334, 763, 328, 27293, 487, 328, 22845, 763, 328, 9997, 13933, 328, 32791, 763, 867, 92, 271, 14162, 25, 15451, 9956, 369, 279, 2531, 303, 30, 198, 3670, 25, 198, 4205, 8, 9665, 198, 5181, 8, 3286, 198, 2929, 8, 4434, 198, 5263, 8, 24863, 198, 15666, 25, 318 ], "slot": 61, "nopts": 4, "label_ids": [ 32, 33, 34, 35 ], "label_strings": [ "A", "B", "C", "D" ], "tokens": 62, "slot_logits": [ 10.723835945129395, 16.095911026000977, 11.311446189880371, 9.567995071411133 ], "p_oracle": [ 0.0053413608111441135, 0.9834700226783752, 0.009449634701013565, 0.001738991355523467 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9740203619003296 }, { "id": "r06-priority", "request_id": "r06", "question_id": "priority", "kind": "list", "type": "choice", "level_index": null, "question": "What is the job priority?", "options": [ "high", "medium", "low" ], "ids": [ 1905, 25, 198, 4754, 8534, 763, 5046, 24344, 763, 328, 18828, 487, 328, 1334, 763, 328, 27293, 487, 328, 22845, 763, 328, 9997, 13933, 328, 32791, 763, 867, 92, 271, 14162, 25, 3437, 369, 279, 2531, 10304, 30, 198, 3670, 25, 198, 4205, 8, 1496, 198, 5181, 8, 10728, 198, 2929, 8, 3238, 198, 15666, 25, 318 ], "slot": 56, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 57, "slot_logits": [ 10.695029258728027, 12.850499153137207, 15.633489608764648 ], "p_oracle": [ 0.007694360800087452, 0.06237579137086868, 0.9299298524856567 ], "argmax": 2, "argmax_label": "C", "argmax_token_id": 34, "top2_margin": 0.8675540685653687 }, { "id": "r06-approved", "request_id": "r06", "question_id": "approved", "kind": "list", "type": "noul", "level_index": null, "question": "Has the job been approved?", "options": [ "no: Approval has not been granted", "yes: Approval has been granted" ], "ids": [ 1905, 25, 198, 4754, 8534, 763, 5046, 24344, 763, 328, 18828, 487, 328, 1334, 763, 328, 27293, 487, 328, 22845, 763, 328, 9997, 13933, 328, 32791, 763, 867, 92, 271, 14162, 25, 11114, 279, 2531, 978, 11454, 30, 198, 3670, 25, 198, 4205, 8, 874, 25, 61625, 682, 524, 978, 11340, 198, 5181, 8, 9542, 25, 61625, 682, 978, 11340, 198, 15666, 25, 318 ], "slot": 63, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 64, "slot_logits": [ 15.264781951904297, 10.294569969177246 ], "p_oracle": [ 0.9920409321784973, 0.007959095761179924 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9840818643569946 }, { "id": "r07-contents", "request_id": "r07", "question_id": "contents", "kind": "list", "type": "choice", "level_index": null, "question": "What does the container hold?", "options": [ "water", "sand", "salt", "seeds" ], "ids": [ 1905, 25, 198, 32, 5303, 9687, 9002, 13, 11116, 25493, 369, 6105, 13, 357, 2306, 2640, 310, 2426, 424, 52239, 11, 321, 279, 24544, 369, 632, 47272, 13, 271, 14162, 25, 3437, 1503, 279, 5303, 3222, 30, 198, 3670, 25, 198, 4205, 8, 2919, 198, 5181, 8, 9002, 198, 2929, 8, 11679, 198, 5263, 8, 18483, 198, 15666, 25, 318 ], "slot": 59, "nopts": 4, "label_ids": [ 32, 33, 34, 35 ], "label_strings": [ "A", "B", "C", "D" ], "tokens": 60, "slot_logits": [ 10.25759220123291, 15.81906509399414, 10.551654815673828, 10.540971755981445 ], "p_oracle": [ 0.0044456240721046925, 0.9837862849235535, 0.005914564710110426, 0.005853535141795874 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9778717160224915 }, { "id": "r07-location", "request_id": "r07", "question_id": "location", "kind": "list", "type": "choice", "level_index": null, "question": "Where should the container be kept?", "options": [ "outdoors", "underground", "indoors", "on a roof", "in a pond" ], "ids": [ 1905, 25, 198, 32, 5303, 9687, 9002, 13, 11116, 25493, 369, 6105, 13, 357, 2306, 2640, 310, 2426, 424, 52239, 11, 321, 279, 24544, 369, 632, 47272, 13, 271, 14162, 25, 10646, 1220, 279, 5303, 381, 8341, 30, 198, 3670, 25, 198, 4205, 8, 32420, 198, 5181, 8, 24458, 198, 2929, 8, 52239, 198, 5263, 8, 383, 264, 14693, 198, 10406, 8, 303, 264, 34393, 198, 15666, 25, 318 ], "slot": 68, "nopts": 5, "label_ids": [ 32, 33, 34, 35, 36 ], "label_strings": [ "A", "B", "C", "D", "E" ], "tokens": 69, "slot_logits": [ 10.422050476074219, 9.836474418640137, 16.16800308227539, 10.209125518798828, 8.553250312805176 ], "p_oracle": [ 0.0037419262807816267, 0.0021192876156419516, 0.9904859662055969, 0.00304310186766088, 0.0006097119185142219 ], "argmax": 2, "argmax_label": "C", "argmax_token_id": 34, "top2_margin": 0.9867440462112427 }, { "id": "r07-seal", "request_id": "r07", "question_id": "seal", "kind": "list", "type": "noul", "level_index": null, "question": "Is the seal broken?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 32, 5303, 9687, 9002, 13, 11116, 25493, 369, 6105, 13, 357, 2306, 2640, 310, 2426, 424, 52239, 11, 321, 279, 24544, 369, 632, 47272, 13, 271, 14162, 25, 2091, 279, 24544, 10548, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 50, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 51, "slot_logits": [ 14.682880401611328, 9.068617820739746 ], "p_oracle": [ 0.9957252740859985, 0.004274771548807621 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9914504885673523 }, { "id": "r08-direction", "request_id": "r08", "question_id": "direction", "kind": "list", "type": "choice", "level_index": null, "question": "Which direction does the route lead?", "options": [ "north", "south", "east", "west", "up" ], "ids": [ 1905, 25, 198, 4754, 8695, 763, 5046, 18498, 763, 328, 10717, 487, 328, 38429, 763, 328, 873, 3747, 487, 328, 32958, 763, 804, 2069, 328, 14764, 763, 328, 7187, 8934, 271, 14162, 25, 15451, 4944, 1503, 279, 5832, 2895, 30, 198, 3670, 25, 198, 4205, 8, 9897, 198, 5181, 8, 9514, 198, 2929, 8, 10663, 198, 5263, 8, 9422, 198, 10406, 8, 685, 198, 15666, 25, 318 ], "slot": 66, "nopts": 5, "label_ids": [ 32, 33, 34, 35, 36 ], "label_strings": [ "A", "B", "C", "D", "E" ], "tokens": 67, "slot_logits": [ 10.893556594848633, 11.913166999816895, 12.04556655883789, 14.447519302368164, 8.572418212890625 ], "p_oracle": [ 0.026061559095978737, 0.07013165950775146, 0.0797516331076622, 0.8213180303573608, 0.002737182192504406 ], "argmax": 3, "argmax_label": "D", "argmax_token_id": 35, "top2_margin": 0.7415664196014404 }, { "id": "r08-surface", "request_id": "r08", "question_id": "surface", "kind": "list", "type": "choice", "level_index": null, "question": "What is the route surface?", "options": [ "gravel", "ice", "asphalt" ], "ids": [ 1905, 25, 198, 4754, 8695, 763, 5046, 18498, 763, 328, 10717, 487, 328, 38429, 763, 328, 873, 3747, 487, 328, 32958, 763, 804, 2069, 328, 14764, 763, 328, 7187, 8934, 271, 14162, 25, 3437, 369, 279, 5832, 7099, 30, 198, 3670, 25, 198, 4205, 8, 40149, 198, 5181, 8, 9561, 198, 2929, 8, 43753, 198, 15666, 25, 318 ], "slot": 57, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 58, "slot_logits": [ 16.040477752685547, 8.846065521240234, 9.934946060180664 ], "p_oracle": [ 0.9964224696159363, 0.0009224751847796142, 0.0026550705078989267 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9937673807144165 }, { "id": "r08-open", "request_id": "r08", "question_id": "open", "kind": "list", "type": "noul", "level_index": null, "question": "Is the route open?", "options": [ "no: The route is closed", "yes: Travel is permitted" ], "ids": [ 1905, 25, 198, 4754, 8695, 763, 5046, 18498, 763, 328, 10717, 487, 328, 38429, 763, 328, 873, 3747, 487, 328, 32958, 763, 804, 2069, 328, 14764, 763, 328, 7187, 8934, 271, 14162, 25, 2091, 279, 5832, 1724, 30, 198, 3670, 25, 198, 4205, 8, 874, 25, 561, 5832, 369, 7629, 198, 5181, 8, 9542, 25, 17558, 369, 14688, 198, 15666, 25, 318 ], "slot": 61, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 62, "slot_logits": [ 15.557018280029297, 10.50631332397461 ], "p_oracle": [ 0.9926348328590393, 0.007365191355347633 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9852696657180786 }, { "id": "r09-liquid", "request_id": "r09", "question_id": "liquid", "kind": "list", "type": "choice", "level_index": null, "question": "What liquid is in the tank?", "options": [ "oil", "water", "ink" ], "ids": [ 1905, 25, 198, 760, 5638, 12526, 369, 4139, 2400, 314, 4101, 2919, 13, 561, 30396, 369, 7629, 13, 11116, 4990, 2119, 369, 6681, 39970, 1881, 4147, 321, 2400, 13, 271, 14162, 25, 3437, 14058, 369, 303, 279, 12526, 30, 198, 3670, 25, 198, 4205, 8, 5415, 198, 5181, 8, 2919, 198, 2929, 8, 25570, 198, 15666, 25, 318 ], "slot": 57, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 58, "slot_logits": [ 10.511930465698242, 15.698722839355469, 10.913081169128418 ], "p_oracle": [ 0.00639850040897727, 0.98415607213974, 0.00944542046636343 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9747106432914734 }, { "id": "r09-valve", "request_id": "r09", "question_id": "valve", "kind": "list", "type": "choice", "level_index": null, "question": "What is the valve position?", "options": [ "open", "closed", "missing" ], "ids": [ 1905, 25, 198, 760, 5638, 12526, 369, 4139, 2400, 314, 4101, 2919, 13, 561, 30396, 369, 7629, 13, 11116, 4990, 2119, 369, 6681, 39970, 1881, 4147, 321, 2400, 13, 271, 14162, 25, 3437, 369, 279, 30396, 2234, 30, 198, 3670, 25, 198, 4205, 8, 1724, 198, 5181, 8, 7629, 198, 2929, 8, 7167, 198, 15666, 25, 318 ], "slot": 56, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 57, "slot_logits": [ 12.051149368286133, 15.182480812072754, 8.624458312988281 ], "p_oracle": [ 0.045570921152830124, 0.9527929425239563, 0.0016362133901566267 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9072220325469971 }, { "id": "r09-clarity", "request_id": "r09", "question_id": "clarity", "kind": "list", "type": "choice", "level_index": null, "question": "How is the water described?", "options": [ "dirty", "cloudy", "frozen", "clean" ], "ids": [ 1905, 25, 198, 760, 5638, 12526, 369, 4139, 2400, 314, 4101, 2919, 13, 561, 30396, 369, 7629, 13, 11116, 4990, 2119, 369, 6681, 39970, 1881, 4147, 321, 2400, 13, 271, 14162, 25, 2500, 369, 279, 2919, 7246, 30, 198, 3670, 25, 198, 4205, 8, 18035, 198, 5181, 8, 71049, 198, 2929, 8, 19088, 198, 5263, 8, 4101, 198, 15666, 25, 318 ], "slot": 60, "nopts": 4, "label_ids": [ 32, 33, 34, 35 ], "label_strings": [ "A", "B", "C", "D" ], "tokens": 61, "slot_logits": [ 9.4433012008667, 8.67347526550293, 8.218214988708496, 16.196657180786133 ], "p_oracle": [ 0.0014170478098094463, 0.0006711058667860925, 0.000431353080784902, 0.9974804520606995 ], "argmax": 3, "argmax_label": "D", "argmax_token_id": 35, "top2_margin": 0.9960634112358093 }, { "id": "r09-fill-level0", "request_id": "r09", "question_id": "fill", "kind": "iso", "type": "score", "level_index": 0, "question": "How full is the tank?\nProposed answer: empty\nDoes the proposed answer fit?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 760, 5638, 12526, 369, 4139, 2400, 314, 4101, 2919, 13, 561, 30396, 369, 7629, 13, 11116, 4990, 2119, 369, 6681, 39970, 1881, 4147, 321, 2400, 13, 271, 14162, 25, 2500, 2400, 369, 279, 12526, 30, 198, 1941, 18988, 4087, 25, 4147, 198, 20810, 279, 10660, 4087, 4786, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 65, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 66, "slot_logits": [ 15.24805736541748, 9.459310531616211 ], "p_oracle": [ 0.9963890314102173, 0.0036110365763306618 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9927780032157898 }, { "id": "r09-fill-level1", "request_id": "r09", "question_id": "fill", "kind": "iso", "type": "score", "level_index": 1, "question": "How full is the tank?\nProposed answer: one quarter full\nDoes the proposed answer fit?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 760, 5638, 12526, 369, 4139, 2400, 314, 4101, 2919, 13, 561, 30396, 369, 7629, 13, 11116, 4990, 2119, 369, 6681, 39970, 1881, 4147, 321, 2400, 13, 271, 14162, 25, 2500, 2400, 369, 279, 12526, 30, 198, 1941, 18988, 4087, 25, 799, 8079, 2400, 198, 20810, 279, 10660, 4087, 4786, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 67, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 68, "slot_logits": [ 13.470892906188965, 12.312257766723633 ], "p_oracle": [ 0.7548943758010864, 0.24510566890239716 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.5097886919975281 }, { "id": "r09-fill-level2", "request_id": "r09", "question_id": "fill", "kind": "iso", "type": "score", "level_index": 2, "question": "How full is the tank?\nProposed answer: half full\nDoes the proposed answer fit?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 760, 5638, 12526, 369, 4139, 2400, 314, 4101, 2919, 13, 561, 30396, 369, 7629, 13, 11116, 4990, 2119, 369, 6681, 39970, 1881, 4147, 321, 2400, 13, 271, 14162, 25, 2500, 2400, 369, 279, 12526, 30, 198, 1941, 18988, 4087, 25, 4139, 2400, 198, 20810, 279, 10660, 4087, 4786, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 66, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 67, "slot_logits": [ 10.861742973327637, 14.389880180358887 ], "p_oracle": [ 0.03151174634695053, 0.9684882164001465 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9369764924049377 }, { "id": "r09-fill-level3", "request_id": "r09", "question_id": "fill", "kind": "iso", "type": "score", "level_index": 3, "question": "How full is the tank?\nProposed answer: three quarters full\nDoes the proposed answer fit?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 760, 5638, 12526, 369, 4139, 2400, 314, 4101, 2919, 13, 561, 30396, 369, 7629, 13, 11116, 4990, 2119, 369, 6681, 39970, 1881, 4147, 321, 2400, 13, 271, 14162, 25, 2500, 2400, 369, 279, 12526, 30, 198, 1941, 18988, 4087, 25, 2250, 30439, 2400, 198, 20810, 279, 10660, 4087, 4786, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 67, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 68, "slot_logits": [ 13.572099685668945, 11.997562408447266 ], "p_oracle": [ 0.821812629699707, 0.17818734049797058 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.6436252593994141 }, { "id": "r09-fill-level4", "request_id": "r09", "question_id": "fill", "kind": "iso", "type": "score", "level_index": 4, "question": "How full is the tank?\nProposed answer: completely full\nDoes the proposed answer fit?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 760, 5638, 12526, 369, 4139, 2400, 314, 4101, 2919, 13, 561, 30396, 369, 7629, 13, 11116, 4990, 2119, 369, 6681, 39970, 1881, 4147, 321, 2400, 13, 271, 14162, 25, 2500, 2400, 369, 279, 12526, 30, 198, 1941, 18988, 4087, 25, 6379, 2400, 198, 20810, 279, 10660, 4087, 4786, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 66, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 67, "slot_logits": [ 14.791322708129883, 9.465218544006348 ], "p_oracle": [ 0.9943530559539795, 0.0056469617411494255 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.988706111907959 }, { "id": "r10-surface", "request_id": "r10", "question_id": "surface", "kind": "list", "type": "choice", "level_index": null, "question": "How is the inspected surface described?", "options": [ "rough", "smooth", "cracked", "sticky" ], "ids": [ 1905, 25, 198, 4754, 51603, 763, 5046, 41447, 763, 328, 6487, 487, 328, 38429, 763, 328, 54964, 487, 328, 3313, 763, 328, 5603, 487, 328, 1767, 763, 220, 19, 3307, 271, 14162, 25, 2500, 369, 279, 59700, 7099, 7246, 30, 198, 3670, 25, 198, 4205, 8, 10841, 198, 5181, 8, 10558, 198, 2929, 8, 41423, 198, 5263, 8, 26515, 198, 15666, 25, 318 ], "slot": 62, "nopts": 4, "label_ids": [ 32, 33, 34, 35 ], "label_strings": [ "A", "B", "C", "D" ], "tokens": 63, "slot_logits": [ 10.968607902526855, 16.204971313476562, 8.701059341430664, 7.75079870223999 ], "p_oracle": [ 0.006152009591460228, 0.9928968548774719, 0.0006806378369219601, 0.00027054722886532545 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9867448210716248 }, { "id": "r10-color", "request_id": "r10", "question_id": "color", "kind": "list", "type": "choice", "level_index": null, "question": "What is the surface color?", "options": [ "black", "white", "gray" ], "ids": [ 1905, 25, 198, 4754, 51603, 763, 5046, 41447, 763, 328, 6487, 487, 328, 38429, 763, 328, 54964, 487, 328, 3313, 763, 328, 5603, 487, 328, 1767, 763, 220, 19, 3307, 271, 14162, 25, 3437, 369, 279, 7099, 1829, 30, 198, 3670, 25, 198, 4205, 8, 3565, 198, 5181, 8, 4022, 198, 2929, 8, 17024, 198, 15666, 25, 318 ], "slot": 57, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 58, "slot_logits": [ 9.81509017944336, 15.177660942077637, 11.372785568237305 ], "p_oracle": [ 0.005320006050169468, 0.9705416560173035, 0.024138320237398148 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9464033246040344 }, { "id": "r10-count", "request_id": "r10", "question_id": "count", "kind": "list", "type": "choice", "level_index": null, "question": "How many items were inspected?", "options": [ "two", "four", "six" ], "ids": [ 1905, 25, 198, 4754, 51603, 763, 5046, 41447, 763, 328, 6487, 487, 328, 38429, 763, 328, 54964, 487, 328, 3313, 763, 328, 5603, 487, 328, 1767, 763, 220, 19, 3307, 271, 14162, 25, 2500, 1599, 3470, 998, 59700, 30, 198, 3670, 25, 198, 4205, 8, 1330, 198, 5181, 8, 2943, 198, 2929, 8, 4590, 198, 15666, 25, 318 ], "slot": 57, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 58, "slot_logits": [ 10.855226516723633, 15.007841110229492, 11.921732902526855 ], "p_oracle": [ 0.016619298607110977, 0.9365748763084412, 0.046805839985609055 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.8897690176963806 }, { "id": "r10-damage-level0", "request_id": "r10", "question_id": "damage", "kind": "iso", "type": "score", "level_index": 0, "question": "How much damage is present?\nProposed answer: no damage\nDoes the proposed answer fit?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 4754, 51603, 763, 5046, 41447, 763, 328, 6487, 487, 328, 38429, 763, 328, 54964, 487, 328, 3313, 763, 328, 5603, 487, 328, 1767, 763, 220, 19, 3307, 271, 14162, 25, 2500, 1691, 5383, 369, 2945, 30, 198, 1941, 18988, 4087, 25, 874, 5383, 198, 20810, 279, 10660, 4087, 4786, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 67, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 68, "slot_logits": [ 9.676294326782227, 14.803475379943848 ], "p_oracle": [ 0.006841752212494612, 0.9931582808494568 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9863165020942688 }, { "id": "r10-damage-level1", "request_id": "r10", "question_id": "damage", "kind": "iso", "type": "score", "level_index": 1, "question": "How much damage is present?\nProposed answer: minor scratches\nDoes the proposed answer fit?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 4754, 51603, 763, 5046, 41447, 763, 328, 6487, 487, 328, 38429, 763, 328, 54964, 487, 328, 3313, 763, 328, 5603, 487, 328, 1767, 763, 220, 19, 3307, 271, 14162, 25, 2500, 1691, 5383, 369, 2945, 30, 198, 1941, 18988, 4087, 25, 8652, 63523, 198, 20810, 279, 10660, 4087, 4786, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 67, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 68, "slot_logits": [ 15.020203590393066, 8.703829765319824 ], "p_oracle": [ 0.9978333115577698, 0.0021666535176336765 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9956666827201843 }, { "id": "r10-damage-level2", "request_id": "r10", "question_id": "damage", "kind": "iso", "type": "score", "level_index": 2, "question": "How much damage is present?\nProposed answer: several shallow dents\nDoes the proposed answer fit?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 4754, 51603, 763, 5046, 41447, 763, 328, 6487, 487, 328, 38429, 763, 328, 54964, 487, 328, 3313, 763, 328, 5603, 487, 328, 1767, 763, 220, 19, 3307, 271, 14162, 25, 2500, 1691, 5383, 369, 2945, 30, 198, 1941, 18988, 4087, 25, 3679, 24785, 220390, 198, 20810, 279, 10660, 4087, 4786, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 68, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 69, "slot_logits": [ 15.100964546203613, 8.283649444580078 ], "p_oracle": [ 0.9986666440963745, 0.0013333152746781707 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9973333477973938 }, { "id": "r10-damage-level3", "request_id": "r10", "question_id": "damage", "kind": "iso", "type": "score", "level_index": 3, "question": "How much damage is present?\nProposed answer: large cracks\nDoes the proposed answer fit?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 4754, 51603, 763, 5046, 41447, 763, 328, 6487, 487, 328, 38429, 763, 328, 54964, 487, 328, 3313, 763, 328, 5603, 487, 328, 1767, 763, 220, 19, 3307, 271, 14162, 25, 2500, 1691, 5383, 369, 2945, 30, 198, 1941, 18988, 4087, 25, 3349, 43369, 198, 20810, 279, 10660, 4087, 4786, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 67, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 68, "slot_logits": [ 15.179399490356445, 8.479437828063965 ], "p_oracle": [ 0.9985060691833496, 0.0014939777320250869 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9970120787620544 }, { "id": "r10-damage-level4", "request_id": "r10", "question_id": "damage", "kind": "iso", "type": "score", "level_index": 4, "question": "How much damage is present?\nProposed answer: completely broken\nDoes the proposed answer fit?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 4754, 51603, 763, 5046, 41447, 763, 328, 6487, 487, 328, 38429, 763, 328, 54964, 487, 328, 3313, 763, 328, 5603, 487, 328, 1767, 763, 220, 19, 3307, 271, 14162, 25, 2500, 1691, 5383, 369, 2945, 30, 198, 1941, 18988, 4087, 25, 6379, 10548, 198, 20810, 279, 10660, 4087, 4786, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 67, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 68, "slot_logits": [ 14.980148315429688, 8.452716827392578 ], "p_oracle": [ 0.9982340335845947, 0.001765925670042634 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9964681267738342 }, { "id": "r11-bin", "request_id": "r11", "question_id": "bin", "kind": "list", "type": "choice", "level_index": null, "question": "Which storage bin is selected?", "options": [ "red", "orange", "yellow", "green", "blue", "indigo", "violet", "black", "white", "gray", "brown" ], "ids": [ 1905, 25, 198, 760, 3955, 5638, 9261, 369, 76990, 13, 11116, 25493, 369, 7629, 13, 271, 14162, 25, 15451, 5638, 9261, 369, 3955, 30, 198, 3670, 25, 198, 7, 32, 8, 2438, 198, 7, 33, 8, 18016, 198, 7, 34, 8, 13358, 198, 7, 35, 8, 5983, 198, 7, 36, 8, 6105, 198, 7, 37, 8, 1214, 7588, 198, 7, 38, 8, 76990, 198, 7, 39, 8, 3565, 198, 7, 40, 8, 4022, 198, 7, 41, 8, 17024, 198, 7, 42, 8, 13477, 198, 15666, 25, 318 ], "slot": 86, "nopts": 11, "label_ids": [ 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42 ], "label_strings": [ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K" ], "tokens": 87, "slot_logits": [ 10.001649856567383, 9.455181121826172, 9.27830696105957, 9.428098678588867, 10.178091049194336, 10.238436698913574, 17.246089935302734, 9.300960540771484, 9.189607620239258, 9.048724174499512, 8.94906997680664 ], "p_oracle": [ 0.0008766235550865531, 0.0005157003179192543, 0.0004343294713180512, 0.0005023173289373517, 0.0010404207278043032, 0.0011031980393454432, 0.9940218329429626, 0.00044398775207810104, 0.0003984920331276953, 0.00034754970693029463, 0.00031549902632832527 ], "argmax": 6, "argmax_label": "G", "argmax_token_id": 38, "top2_margin": 0.9929186105728149 }, { "id": "r12-slot", "request_id": "r12", "question_id": "slot", "kind": "list", "type": "choice", "level_index": null, "question": "Which archive slot is requested?", "options": [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "100", "101", "102", "103", "104", "105", "106", "107", "108", "109", "110", "111", "112", "113", "114", "115", "116", "117", "118", "119", "120", "121", "122", "123", "124", "125", "126", "127", "128", "129", "130", "131", "132", "133", "134", "135", "136", "137", "138", "139", "140", "141", "142", "143", "144", "145", "146", "147", "148", "149", "150", "151", "152", "153", "154", "155", "156", "157", "158", "159", "160", "161", "162", "163", "164", "165", "166", "167", "168", "169", "170", "171", "172", "173", "174", "175", "176", "177", "178", "179", "180", "181", "182", "183", "184", "185", "186", "187", "188", "189", "190", "191", "192", "193", "194", "195", "196", "197", "198", "199", "200", "201", "202", "203", "204", "205", "206", "207", "208", "209", "210", "211", "212", "213", "214", "215", "216", "217", "218", "219", "220", "221", "222", "223", "224", "225", "226", "227", "228", "229", "230", "231", "232", "233", "234", "235", "236", "237", "238", "239", "240", "241", "242", "243", "244", "245", "246", "247", "248", "249", "250", "251", "252", "253", "254", "255" ], "ids": [ 1905, 25, 198, 760, 10897, 17593, 9167, 369, 220, 16, 22, 18, 13, 30523, 421, 9167, 13, 271, 14162, 25, 15451, 17593, 9167, 369, 10897, 30, 198, 3670, 25, 198, 7, 32, 8, 220, 16, 198, 7, 33, 8, 220, 17, 198, 7, 34, 8, 220, 18, 198, 7, 35, 8, 220, 19, 198, 7, 36, 8, 220, 20, 198, 7, 37, 8, 220, 21, 198, 7, 38, 8, 220, 22, 198, 7, 39, 8, 220, 23, 198, 7, 40, 8, 220, 24, 198, 7, 41, 8, 220, 16, 15, 198, 7, 42, 8, 220, 16, 16, 198, 7, 43, 8, 220, 16, 17, 198, 7, 44, 8, 220, 16, 18, 198, 7, 45, 8, 220, 16, 19, 198, 7, 46, 8, 220, 16, 20, 198, 7, 47, 8, 220, 16, 21, 198, 7, 48, 8, 220, 16, 22, 198, 7, 49, 8, 220, 16, 23, 198, 7, 50, 8, 220, 16, 24, 198, 7, 51, 8, 220, 17, 15, 198, 7, 52, 8, 220, 17, 16, 198, 7, 53, 8, 220, 17, 17, 198, 7, 54, 8, 220, 17, 18, 198, 7, 55, 8, 220, 17, 19, 198, 7, 56, 8, 220, 17, 20, 198, 7, 57, 8, 220, 17, 21, 198, 7, 5840, 8, 220, 17, 22, 198, 7, 1803, 8, 220, 17, 23, 198, 7, 1646, 8, 220, 17, 24, 198, 7, 1745, 8, 220, 18, 15, 198, 7, 13276, 8, 220, 18, 16, 198, 7, 8018, 8, 220, 18, 17, 198, 7, 1825, 8, 220, 18, 18, 198, 7, 28946, 8, 220, 18, 19, 198, 7, 15015, 8, 220, 18, 20, 198, 7, 29595, 8, 220, 18, 21, 198, 7, 11568, 8, 220, 18, 22, 198, 7, 939, 8, 220, 18, 23, 198, 7, 1354, 8, 220, 18, 24, 198, 7, 1058, 8, 220, 19, 15, 198, 7, 18183, 8, 220, 19, 16, 198, 7, 2456, 8, 220, 19, 17, 198, 7, 88898, 8, 220, 19, 18, 198, 7, 905, 8, 220, 19, 19, 198, 7, 1846, 8, 220, 19, 20, 198, 7, 802, 8, 220, 19, 21, 198, 7, 33869, 8, 220, 19, 22, 198, 7, 7839, 8, 220, 19, 23, 198, 7, 14006, 8, 220, 19, 24, 198, 7, 2860, 8, 220, 20, 15, 198, 7, 2926, 8, 220, 20, 16, 198, 7, 22828, 8, 220, 20, 17, 198, 7, 6844, 8, 220, 20, 18, 198, 7, 9798, 8, 220, 20, 19, 198, 7, 4738, 8, 220, 20, 20, 198, 7, 9265, 8, 220, 20, 21, 198, 7, 11261, 8, 220, 20, 22, 198, 7, 19278, 8, 220, 20, 23, 198, 7, 36513, 8, 220, 20, 24, 198, 7, 93801, 8, 220, 21, 15, 198, 7, 8335, 8, 220, 21, 16, 198, 7, 14544, 8, 220, 21, 17, 198, 7, 85266, 8, 220, 21, 18, 198, 7, 9110, 8, 220, 21, 19, 198, 7, 28000, 8, 220, 21, 20, 198, 7, 15137, 8, 220, 21, 21, 198, 7, 4525, 8, 220, 21, 22, 198, 7, 25261, 8, 220, 21, 23, 198, 7, 12717, 8, 220, 21, 24, 198, 7, 7116, 8, 220, 22, 15, 198, 7, 17078, 8, 220, 22, 16, 198, 7, 14497, 8, 220, 22, 17, 198, 7, 57339, 8, 220, 22, 18, 198, 7, 74909, 8, 220, 22, 19, 198, 7, 52072, 8, 220, 22, 20, 198, 7, 19305, 8, 220, 22, 21, 198, 7, 4887, 8, 220, 22, 22, 198, 7, 12607, 8, 220, 22, 23, 198, 7, 3580, 8, 220, 22, 24, 198, 7, 6281, 8, 220, 23, 15, 198, 7, 2036, 8, 220, 23, 16, 198, 7, 9362, 8, 220, 23, 17, 198, 7, 8533, 8, 220, 23, 18, 198, 7, 2080, 8, 220, 23, 19, 198, 7, 10911, 8, 220, 23, 20, 198, 7, 2925, 8, 220, 23, 21, 198, 7, 3040, 8, 220, 23, 22, 198, 7, 9690, 8, 220, 23, 23, 198, 7, 27731, 8, 220, 23, 24, 198, 7, 8023, 8, 220, 24, 15, 198, 7, 6901, 8, 220, 24, 16, 198, 7, 8702, 8, 220, 24, 17, 198, 7, 6211, 8, 220, 24, 18, 198, 7, 1123, 8, 220, 24, 19, 198, 7, 16307, 8, 220, 24, 20, 198, 7, 18990, 8, 220, 24, 21, 198, 7, 64045, 8, 220, 24, 22, 198, 7, 63037, 8, 220, 24, 23, 198, 7, 33380, 8, 220, 24, 24, 198, 7, 6151, 8, 220, 16, 15, 15, 198, 7, 3392, 8, 220, 16, 15, 16, 198, 7, 5449, 8, 220, 16, 15, 17, 198, 7, 3967, 8, 220, 16, 15, 18, 198, 7, 1113, 8, 220, 16, 15, 19, 198, 7, 5095, 8, 220, 16, 15, 20, 198, 7, 51923, 8, 220, 16, 15, 21, 198, 7, 49600, 8, 220, 16, 15, 22, 198, 7, 17099, 8, 220, 16, 15, 23, 198, 7, 51483, 8, 220, 16, 15, 24, 198, 7, 17756, 8, 220, 16, 16, 15, 198, 7, 16037, 8, 220, 16, 16, 16, 198, 7, 8135, 8, 220, 16, 16, 17, 198, 7, 30237, 8, 220, 16, 16, 18, 198, 7, 5683, 8, 220, 16, 16, 19, 198, 7, 9992, 8, 220, 16, 16, 20, 198, 7, 7444, 8, 220, 16, 16, 21, 198, 7, 5751, 8, 220, 16, 16, 22, 198, 7, 10284, 8, 220, 16, 16, 23, 198, 7, 20887, 8, 220, 16, 16, 24, 198, 7, 59884, 8, 220, 16, 17, 15, 198, 7, 52396, 8, 220, 16, 17, 16, 198, 7, 16103, 8, 220, 16, 17, 17, 198, 7, 67547, 8, 220, 16, 17, 18, 198, 7, 18535, 8, 220, 16, 17, 19, 198, 7, 8006, 8, 220, 16, 17, 20, 198, 7, 7263, 8, 220, 16, 17, 21, 198, 7, 1425, 8, 220, 16, 17, 22, 198, 7, 6878, 8, 220, 16, 17, 23, 198, 7, 14453, 8, 220, 16, 17, 24, 198, 7, 9097, 8, 220, 16, 18, 15, 198, 7, 44072, 8, 220, 16, 18, 16, 198, 7, 76089, 8, 220, 16, 18, 17, 198, 7, 68720, 8, 220, 16, 18, 18, 198, 7, 2662, 8, 220, 16, 18, 19, 198, 7, 2629, 8, 220, 16, 18, 20, 198, 7, 923, 8, 220, 16, 18, 21, 198, 7, 6548, 8, 220, 16, 18, 22, 198, 7, 8924, 8, 220, 16, 18, 23, 198, 7, 52194, 8, 220, 16, 18, 24, 198, 7, 622, 8, 220, 16, 19, 15, 198, 7, 1515, 8, 220, 16, 19, 16, 198, 7, 1300, 8, 220, 16, 19, 17, 198, 7, 37523, 8, 220, 16, 19, 18, 198, 7, 44473, 8, 220, 16, 19, 19, 198, 7, 36530, 8, 220, 16, 19, 20, 198, 7, 3152, 8, 220, 16, 19, 21, 198, 7, 93924, 8, 220, 16, 19, 22, 198, 7, 3505, 8, 220, 16, 19, 23, 198, 7, 15731, 8, 220, 16, 19, 24, 198, 7, 6542, 8, 220, 16, 20, 15, 198, 7, 14176, 8, 220, 16, 20, 16, 198, 7, 11091, 8, 220, 16, 20, 17, 198, 7, 1686, 8, 220, 16, 20, 18, 198, 7, 11660, 8, 220, 16, 20, 19, 198, 7, 80440, 8, 220, 16, 20, 20, 198, 7, 18836, 8, 220, 16, 20, 21, 198, 7, 26132, 8, 220, 16, 20, 22, 198, 7, 5934, 8, 220, 16, 20, 23, 198, 7, 24794, 8, 220, 16, 20, 24, 198, 7, 40229, 8, 220, 16, 21, 15, 198, 7, 3660, 8, 220, 16, 21, 16, 198, 7, 11361, 8, 220, 16, 21, 17, 198, 7, 10191, 8, 220, 16, 21, 18, 198, 7, 8225, 8, 220, 16, 21, 19, 198, 7, 3860, 8, 220, 16, 21, 20, 198, 7, 78413, 8, 220, 16, 21, 21, 198, 7, 17680, 8, 220, 16, 21, 22, 198, 7, 15900, 8, 220, 16, 21, 23, 198, 7, 78138, 8, 220, 16, 21, 24, 198, 7, 15653, 8, 220, 16, 22, 15, 198, 7, 5213, 8, 220, 16, 22, 16, 198, 7, 22150, 8, 220, 16, 22, 17, 198, 7, 39500, 8, 220, 16, 22, 18, 198, 7, 10460, 8, 220, 16, 22, 19, 198, 7, 35131, 8, 220, 16, 22, 20, 198, 7, 21563, 8, 220, 16, 22, 21, 198, 7, 43194, 8, 220, 16, 22, 22, 198, 7, 27127, 8, 220, 16, 22, 23, 198, 7, 3697, 8, 220, 16, 22, 24, 198, 7, 20011, 8, 220, 16, 23, 15, 198, 7, 24368, 8, 220, 16, 23, 16, 198, 7, 15058, 8, 220, 16, 23, 17, 198, 7, 23658, 8, 220, 16, 23, 18, 198, 7, 8362, 8, 220, 16, 23, 19, 198, 7, 16035, 8, 220, 16, 23, 20, 198, 7, 24583, 8, 220, 16, 23, 21, 198, 7, 52857, 8, 220, 16, 23, 22, 198, 7, 38403, 8, 220, 16, 23, 23, 198, 7, 60456, 8, 220, 16, 23, 24, 198, 7, 81519, 8, 220, 16, 24, 15, 198, 7, 40097, 8, 220, 16, 24, 16, 198, 7, 16522, 8, 220, 16, 24, 17, 198, 7, 29722, 8, 220, 16, 24, 18, 198, 7, 21756, 8, 220, 16, 24, 19, 198, 7, 18567, 8, 220, 16, 24, 20, 198, 7, 1736, 8, 220, 16, 24, 21, 198, 7, 48043, 8, 220, 16, 24, 22, 198, 7, 87013, 8, 220, 16, 24, 23, 198, 7, 22456, 8, 220, 16, 24, 24, 198, 7, 23165, 8, 220, 17, 15, 15, 198, 7, 55523, 8, 220, 17, 15, 16, 198, 7, 13097, 8, 220, 17, 15, 17, 198, 7, 50397, 8, 220, 17, 15, 18, 198, 7, 41741, 8, 220, 17, 15, 19, 198, 7, 23073, 8, 220, 17, 15, 20, 198, 7, 6401, 8, 220, 17, 15, 21, 198, 7, 86538, 8, 220, 17, 15, 22, 198, 7, 16585, 8, 220, 17, 15, 23, 198, 7, 11622, 8, 220, 17, 15, 24, 198, 7, 2464, 8, 220, 17, 16, 15, 198, 7, 84982, 8, 220, 17, 16, 16, 198, 7, 75516, 8, 220, 17, 16, 17, 198, 7, 36984, 8, 220, 17, 16, 18, 198, 7, 58795, 8, 220, 17, 16, 19, 198, 7, 47217, 8, 220, 17, 16, 20, 198, 7, 59675, 8, 220, 17, 16, 21, 198, 7, 5681, 8, 220, 17, 16, 22, 198, 7, 3151, 8, 220, 17, 16, 23, 198, 7, 1271, 8, 220, 17, 16, 24, 198, 7, 887, 8, 220, 17, 17, 15, 198, 7, 5203, 8, 220, 17, 17, 16, 198, 7, 2685, 8, 220, 17, 17, 17, 198, 7, 1849, 8, 220, 17, 17, 18, 198, 7, 72470, 8, 220, 17, 17, 19, 198, 7, 5370, 8, 220, 17, 17, 20, 198, 7, 74063, 8, 220, 17, 17, 21, 198, 7, 27629, 8, 220, 17, 17, 22, 198, 7, 1655, 8, 220, 17, 17, 23, 198, 7, 1728, 8, 220, 17, 17, 24, 198, 7, 669, 8, 220, 17, 18, 15, 198, 7, 3682, 8, 220, 17, 18, 16, 198, 7, 3191, 8, 220, 17, 18, 17, 198, 7, 59865, 8, 220, 17, 18, 18, 198, 7, 2712, 8, 220, 17, 18, 19, 198, 7, 1580, 8, 220, 17, 18, 20, 198, 7, 922, 8, 220, 17, 18, 21, 198, 7, 77243, 8, 220, 17, 18, 22, 198, 7, 2990, 8, 220, 17, 18, 23, 198, 7, 78493, 8, 220, 17, 18, 24, 198, 7, 5228, 8, 220, 17, 19, 15, 198, 7, 2750, 8, 220, 17, 19, 16, 198, 7, 42711, 8, 220, 17, 19, 17, 198, 7, 44568, 8, 220, 17, 19, 18, 198, 7, 56402, 8, 220, 17, 19, 19, 198, 7, 48236, 8, 220, 17, 19, 20, 198, 7, 38993, 8, 220, 17, 19, 21, 198, 7, 43559, 8, 220, 17, 19, 22, 198, 7, 61250, 8, 220, 17, 19, 23, 198, 7, 32942, 8, 220, 17, 19, 24, 198, 7, 86302, 8, 220, 17, 20, 15, 198, 7, 25310, 8, 220, 17, 20, 16, 198, 7, 26313, 8, 220, 17, 20, 17, 198, 7, 81770, 8, 220, 17, 20, 18, 198, 7, 12185, 8, 220, 17, 20, 19, 198, 7, 78382, 8, 220, 17, 20, 20, 198, 15666, 25, 318 ], "slot": 1964, "nopts": 255, "label_ids": [ 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 5840, 1803, 1646, 1745, 13276, 8018, 1825, 28946, 15015, 29595, 11568, 939, 1354, 1058, 18183, 2456, 88898, 905, 1846, 802, 33869, 7839, 14006, 2860, 2926, 22828, 6844, 9798, 4738, 9265, 11261, 19278, 36513, 93801, 8335, 14544, 85266, 9110, 28000, 15137, 4525, 25261, 12717, 7116, 17078, 14497, 57339, 74909, 52072, 19305, 4887, 12607, 3580, 6281, 2036, 9362, 8533, 2080, 10911, 2925, 3040, 9690, 27731, 8023, 6901, 8702, 6211, 1123, 16307, 18990, 64045, 63037, 33380, 6151, 3392, 5449, 3967, 1113, 5095, 51923, 49600, 17099, 51483, 17756, 16037, 8135, 30237, 5683, 9992, 7444, 5751, 10284, 20887, 59884, 52396, 16103, 67547, 18535, 8006, 7263, 1425, 6878, 14453, 9097, 44072, 76089, 68720, 2662, 2629, 923, 6548, 8924, 52194, 622, 1515, 1300, 37523, 44473, 36530, 3152, 93924, 3505, 15731, 6542, 14176, 11091, 1686, 11660, 80440, 18836, 26132, 5934, 24794, 40229, 3660, 11361, 10191, 8225, 3860, 78413, 17680, 15900, 78138, 15653, 5213, 22150, 39500, 10460, 35131, 21563, 43194, 27127, 3697, 20011, 24368, 15058, 23658, 8362, 16035, 24583, 52857, 38403, 60456, 81519, 40097, 16522, 29722, 21756, 18567, 1736, 48043, 87013, 22456, 23165, 55523, 13097, 50397, 41741, 23073, 6401, 86538, 16585, 11622, 2464, 84982, 75516, 36984, 58795, 47217, 59675, 5681, 3151, 1271, 887, 5203, 2685, 1849, 72470, 5370, 74063, 27629, 1655, 1728, 669, 3682, 3191, 59865, 2712, 1580, 922, 77243, 2990, 78493, 5228, 2750, 42711, 44568, 56402, 48236, 38993, 43559, 61250, 32942, 86302, 25310, 26313, 81770, 12185, 78382 ], "label_strings": [ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "AA", "AB", "AC", "AD", "AE", "AF", "AG", "AH", "AI", "AJ", "AK", "AL", "AM", "AN", "AO", "AP", "AQ", "AR", "AS", "AT", "AU", "AV", "AW", "AX", "AY", "AZ", "BA", "BB", "BC", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BK", "BL", "BM", "BN", "BO", "BP", "BR", "BS", "BT", "BU", "BV", "BW", "BX", "BY", "CA", "CB", "CC", "CD", "CE", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CP", "CR", "CS", "CT", "CU", "CV", "CW", "CX", "CY", "DA", "DB", "DC", "DD", "DE", "DF", "DG", "DH", "DI", "DJ", "DK", "DL", "DM", "DN", "DO", "DP", "DR", "DS", "DT", "DU", "DV", "DW", "DX", "DY", "EA", "EB", "EC", "ED", "EE", "EF", "EG", "EH", "EI", "EK", "EL", "EM", "EN", "EO", "EP", "EQ", "ER", "ES", "ET", "EU", "EV", "EW", "EX", "EZ", "FA", "FB", "FC", "FD", "FE", "FF", "FG", "FH", "FI", "FK", "FL", "FM", "FN", "FO", "FP", "FR", "FS", "FT", "FU", "FW", "FX", "FY", "GA", "GB", "GC", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GO", "GP", "GR", "GS", "GT", "GU", "GV", "GW", "GX", "GY", "HA", "HB", "HC", "HD", "HE", "HF", "HG", "HH", "HI", "HK", "HL", "HM", "HN", "HO", "HP", "HQ", "HR", "HS", "HT", "HU", "HV", "HW", "HX", "HY", "HZ", "IA", "IB", "IC", "ID", "IE", "IF", "IG", "IH", "II", "IJ", "IK", "IL", "IM", "IN", "IO", "IP", "IQ", "IR", "IS", "IT", "IU", "IV", "IW", "IX", "IZ", "JA", "JB", "JC", "JD", "JE", "JI", "JJ", "JK", "JM", "JO", "JP", "JR", "JS", "JT" ], "tokens": 1965, "slot_logits": [ 8.884895324707031, 7.109546661376953, 8.99267292022705, 11.102703094482422, 6.715926170349121, 7.028192520141602, 12.69255256652832, 5.511425495147705, 7.440581321716309, 6.19407844543457, 7.46565580368042, 8.24598503112793, 9.13496208190918, 8.637779235839844, 8.949962615966797, 9.692554473876953, 15.081480026245117, 9.767149925231934, 8.010605812072754, 7.023630619049072, 5.543872833251953, 6.423668384552002, 5.913975715637207, 7.978014945983887, 6.822565078735352, 6.56857442855835, 5.395960807800293, 5.903054237365723, 6.732303619384766, 10.269546508789062, 6.935548305511475, 4.325345993041992, 9.482969284057617, 4.279327392578125, 6.8344292640686035, 3.4362382888793945, 4.918243408203125, 5.4705095291137695, 5.3077921867370605, 4.954349517822266, 7.977176666259766, 5.596954345703125, 10.576948165893555, 6.183475971221924, 6.931347846984863, 5.616436004638672, 6.34905481338501, 3.530465602874756, 3.9203691482543945, 4.811446189880371, 5.99796199798584, 5.208022594451904, 6.381072998046875, 4.1881303787231445, 6.288707733154297, 10.95864486694336, 4.835454940795898, 4.081417560577393, 8.956929206848145, 4.5123186111450195, 3.1871938705444336, 5.119731903076172, 4.685849666595459, 4.261919021606445, 4.45442008972168, 4.1658854484558105, 6.877104759216309, 5.599338531494141, 6.022534370422363, 4.964568138122559, 4.193175792694092, 4.2647294998168945, 4.496939659118652, 5.503272533416748, 5.134926795959473, 6.134653568267822, 6.960299968719482, 6.0316996574401855, 5.400303363800049, 11.87009048461914, 5.892451286315918, 5.308135509490967, 10.24020004272461, 6.794942855834961, 6.563873767852783, 5.087848663330078, 6.4968342781066895, 5.601196765899658, 6.065832614898682, 7.831340312957764, 5.4508256912231445, 7.0425310134887695, 7.145572662353516, 7.056839942932129, 4.6734232902526855, 4.584314346313477, 5.637223720550537, 5.326372146606445, 4.852431297302246, 9.631336212158203, 10.021059036254883, 9.098381042480469, 9.604057312011719, 9.538771629333496, 8.319412231445312, 12.568214416503906, 8.072932243347168, 7.709135055541992, 8.355192184448242, 8.656790733337402, 9.321187019348145, 8.903663635253906, 8.14959716796875, 10.278278350830078, 9.755019187927246, 8.562744140625, 10.43995475769043, 8.19881534576416, 9.159713745117188, 6.137361526489258, 8.357068061828613, 9.030877113342285, 8.005996704101562, 6.427003383636475, 3.713324546813965, 4.007176876068115, 10.213618278503418, 5.945363521575928, 3.4722740650177, 8.381196022033691, 3.5251712799072266, 4.3141632080078125, 3.412759780883789, 3.2190608978271484, 3.1448357105255127, 4.107089996337891, 7.87857723236084, 4.577111721038818, 9.2583646774292, 4.665196895599365, 5.792167663574219, 4.554849624633789, 4.021178245544434, 3.480125665664673, 3.833954334259033, 3.8293309211730957, 3.3508081436157227, 3.9337737560272217, 3.6365365982055664, 5.054380416870117, 9.677553176879883, 4.199941635131836, 3.594815731048584, 8.53744125366211, 3.7327213287353516, 2.25238037109375, 3.8109683990478516, 3.078308582305908, 3.675778865814209, 2.864805221557617, 6.955702781677246, 4.376712322235107, 4.252712249755859, 3.6330790519714355, 4.868338584899902, 3.999619960784912, 3.094320297241211, 4.24783182144165, 5.079629421234131, 12.605399131774902, 11.841745376586914, 12.850505828857422, 17.18320083618164, 12.285043716430664, 10.099553108215332, 11.358125686645508, 9.645862579345703, 10.295759201049805, 10.921961784362793, 11.519821166992188, 10.612663269042969, 11.300433158874512, 11.053281784057617, 10.76918888092041, 11.691640853881836, 10.25477409362793, 10.00456714630127, 10.320024490356445, 10.528053283691406, 9.794836044311523, 8.395038604736328, 3.870840549468994, 3.668914318084717, 3.7110376358032227, 10.614351272583008, 4.054531097412109, 3.8234643936157227, 7.855389595031738, 3.4940457344055176, 2.8840670585632324, 2.9056124687194824, 3.4537980556488037, 3.691086530685425, 3.928797721862793, 5.590273857116699, 4.655778884887695, 10.483772277832031, 4.834507942199707, 4.95920467376709, 3.747077465057373, 3.4806511402130127, 3.5398685932159424, 3.6488850116729736, 4.574560165405273, 2.5029196739196777, 3.2798428535461426, 5.113586902618408, 3.4378440380096436, 5.236751079559326, 9.923702239990234, 5.640061378479004, 4.656484127044678, 7.017945289611816, 2.717491865158081, 4.3957624435424805, 4.125661849975586, 2.518566370010376, 4.599504470825195, 2.649177074432373, 5.458998203277588, 6.449730396270752, 3.510647773742676, 8.776832580566406, 4.371769905090332, 5.386251449584961, 2.6834793090820312, 3.991337537765503, 5.808228015899658, 2.694892406463623, 4.766252517700195, 4.000808238983154, 3.1118812561035156, 2.9596357345581055, 4.111440658569336, 9.82513427734375, 2.642766237258911, 2.1589343547821045, 4.1785407066345215, 2.959895133972168, 2.5606577396392822, 5.029175758361816, 4.014326095581055, 4.480198383331299, 5.115286350250244, 3.482922077178955 ], "p_oracle": [ 0.00025002742768265307, 4.46088015451096e-05, 0.0002776079054456204, 0.002153354696929455, 3.0440554837696254e-05, 4.12209628848359e-05, 0.010080167092382908, 9.45323881751392e-06, 6.151765410322696e-05, 1.8340773749514483e-05, 6.303364352788776e-05, 0.000134460351546295, 0.0003187331312801689, 0.0001966945274034515, 0.00026633183006197214, 0.0005476878141053021, 0.10250557959079742, 0.0005888246232643723, 0.00010699114500312135, 4.10388020100072e-05, 9.755772225616965e-06, 2.2920459741726518e-05, 1.397377945977496e-05, 0.00010365874186391011, 3.376107633812353e-05, 2.638287332956679e-05, 8.450755558442324e-06, 1.3826399481331464e-05, 3.092841507168487e-05, 0.0009590017143636942, 3.7675163184758276e-05, 2.9886575703130802e-06, 0.00044685081229545176, 2.8580673188116634e-06, 3.415219907765277e-05, 1.2606312793650432e-06, 5.314572263159789e-06, 9.08507627173094e-06, 7.757460480206646e-06, 5.504174168891041e-06, 0.0001035744498949498, 1.0271732207911555e-05, 0.0012925089104101062, 1.815296127460897e-05, 3.7521804188145325e-05, 1.0467853826412465e-05, 2.1318810468073934e-05, 1.3813976238452597e-06, 2.0170630250504473e-06, 4.791125320480205e-06, 1.5160945622483268e-05, 7.041288881737273e-06, 2.1991934772813693e-05, 2.6158904802286997e-06, 2.0105630028410815e-05, 0.0018722949316725135, 4.904116849502316e-06, 2.3584395876241615e-06, 0.0002681392652448267, 3.5835375911119627e-06, 9.898735697788652e-07, 6.462861165346112e-06, 4.241120677761501e-06, 2.8101662792323623e-06, 3.38765858032275e-06, 2.560000893936376e-06, 3.559692777344026e-05, 1.0295524589309935e-05, 1.5526977222179994e-05, 5.55905353394337e-06, 2.6287370928912424e-06, 2.8178467346151592e-06, 3.5304301491123624e-06, 9.378707545693032e-06, 6.558910627063597e-06, 1.7312577256234363e-05, 3.859149001073092e-05, 1.5665762475691736e-05, 8.486460501444526e-06, 0.004536079242825508, 1.3684801160707138e-05, 7.760050721117295e-06, 0.000932063499931246, 3.2867726986296475e-05, 2.6262752726324834e-05, 6.265872343647061e-06, 2.4607821615063585e-05, 1.031411829899298e-05, 1.6193613191717304e-05, 8.990032802103087e-05, 8.913102647056803e-06, 4.179880124866031e-05, 4.6196688344934955e-05, 4.2383529944345355e-05, 4.190259005554253e-06, 3.84298346034484e-06, 1.0681268577172887e-05, 7.898664989625104e-06, 4.9856180339702405e-06, 0.0005160845466889441, 0.0007534350734204054, 0.00030761174275539815, 0.0005025957361795008, 0.00047172774793580174, 0.0001443958462914452, 0.008933906443417072, 0.00011366524995537475, 7.984249532455578e-05, 0.0001494999451097101, 0.00020035890338476747, 0.0003818984259851277, 0.0002546251635067165, 0.00012244832760188729, 0.000967165979091078, 0.0005819307989440858, 0.00018287496641278267, 0.0011315427254885435, 0.0001284415484406054, 0.0003264852857682854, 1.7358141121803783e-05, 0.00014977251703385264, 0.0002880980900954455, 0.0001065133765223436, 2.2994790924713016e-05, 1.6497568822160247e-06, 2.1944283616903704e-06, 0.000908317044377327, 1.4406166883418337e-05, 1.305515752392239e-06, 0.00015332236944232136, 1.3743150475420407e-06, 2.9563834686996415e-06, 1.2322205975578981e-06, 1.0209770380242844e-06, 9.499912607680017e-07, 2.4179607862606645e-06, 9.411927021574229e-05, 3.81620566258789e-06, 0.0003593016881495714, 4.156928753218381e-06, 1.2415222045092378e-05, 3.7346069348132005e-06, 2.224462150479667e-06, 1.3155066653780523e-06, 1.8547406170910108e-06, 1.8464334061718546e-06, 1.1602907079577562e-06, 2.0434843008843018e-06, 1.5312392633859417e-06, 6.065543402655749e-06, 0.0005397693603299558, 2.6460618300916394e-06, 1.470453639740299e-06, 0.00017843724344857037, 1.6811198975119623e-06, 3.994110215899127e-07, 1.8138069890483166e-06, 8.905714139473275e-07, 1.590703959664097e-06, 7.238458010760951e-07, 3.841959551209584e-05, 3.141479282930959e-06, 2.7851613140228437e-06, 1.5261075532180257e-06, 5.0632124839467e-06, 2.1783876036352012e-06, 9.045234037330374e-07, 2.7719968329620315e-06, 6.216067049535923e-06, 0.00926232524216175, 0.0044129518792033195, 0.011750814504921436, 0.7887296080589294, 0.006786466110497713, 0.0008130966452881694, 0.002759390976279974, 0.0005234149284660816, 0.000983720296062529, 0.0018067867495119572, 0.003228430636227131, 0.0013381121680140495, 0.0026090804021805525, 0.0020524715073406696, 0.00155772571451962, 0.003814505413174629, 0.0009453455568291247, 0.0007414669962599874, 0.0010071705328300595, 0.0012325868010520935, 0.0006048671202734113, 0.00015539680316578597, 1.9223650724597974e-06, 1.5801366544110351e-06, 1.6460982124044676e-06, 0.0013403076445683837, 2.2976716991252033e-06, 1.8359454543315223e-06, 9.20241727726534e-05, 1.333405748482619e-06, 7.37509481041343e-07, 7.530991297244327e-07, 1.282306584471371e-06, 1.614519987924723e-06, 2.0336372017482063e-06, 1.0205317266809288e-05, 4.1190914998878725e-06, 0.0011807188857346773, 4.89960984850768e-06, 5.530182079382939e-06, 1.7047153733074083e-06, 1.3161767355995835e-06, 1.3940655207989039e-06, 1.5497056438107393e-06, 3.8067626064730575e-06, 5.0939985385412e-07, 1.0830405017259181e-06, 6.424417733796872e-06, 1.262598516404978e-06, 7.240449576784158e-06, 0.0006854813545942307, 1.0710738024499733e-05, 4.121913207200123e-06, 4.0812898078002036e-05, 6.273821213653719e-07, 3.200123273927602e-06, 2.461954636601149e-06, 5.171972929929325e-07, 3.900078354490688e-06, 5.871208372809633e-07, 8.984105079434812e-06, 2.3507813239120878e-05, 1.3550717312682536e-06, 0.00022512504074256867, 3.126439878542442e-06, 8.371463991352357e-06, 6.070031304261647e-07, 2.1609405393974157e-06, 1.2610324120032601e-05, 6.137665309324802e-07, 4.585451733873924e-06, 2.1809028112329543e-06, 9.200776389661769e-07, 7.936535553199064e-07, 2.4281953301397152e-06, 0.0006229236023500562, 5.834775720359175e-07, 3.6477001685852883e-07, 2.591647671579267e-06, 7.93853416780621e-07, 5.387703367887298e-07, 5.918919214309426e-06, 2.2097124201536644e-06, 3.4735091958282283e-06, 6.435025625251001e-06, 1.3190820027375594e-06 ], "argmax": 172, "argmax_label": "GD", "argmax_token_id": 39500, "top2_margin": 0.6862240433692932 }, { "id": "r13-status", "request_id": "r13", "question_id": "status", "kind": "list", "type": "choice", "level_index": null, "question": "What is the current status?", "options": [ "running", "paused", "finished" ], "ids": [ 1905, 25, 198, 4754, 22821, 763, 5046, 2738, 763, 328, 75608, 487, 328, 19290, 763, 328, 51603, 13933, 328, 48528, 40758, 763, 867, 92, 271, 14162, 25, 3437, 369, 279, 1428, 2552, 30, 198, 3670, 25, 198, 4205, 8, 4162, 198, 5181, 8, 33351, 198, 2929, 8, 7807, 198, 15666, 25, 318 ], "slot": 51, "nopts": 3, "label_ids": [ 32, 33, 34 ], "label_strings": [ "A", "B", "C" ], "tokens": 52, "slot_logits": [ 9.794645309448242, 16.4602108001709, 9.169466972351074 ], "p_oracle": [ 0.001543335267342627, 0.9976155757904053, 0.0008411174057982862 ], "argmax": 1, "argmax_label": "B", "argmax_token_id": 33, "top2_margin": 0.9960722327232361 }, { "id": "r13-restart", "request_id": "r13", "question_id": "restart", "kind": "list", "type": "noul", "level_index": null, "question": "Is a restart currently allowed?", "options": [ "no", "yes" ], "ids": [ 1905, 25, 198, 4754, 22821, 763, 5046, 2738, 763, 328, 75608, 487, 328, 19290, 763, 328, 51603, 13933, 328, 48528, 40758, 763, 867, 92, 271, 14162, 25, 2091, 264, 16526, 4862, 5252, 30, 198, 3670, 25, 198, 4205, 8, 874, 198, 5181, 8, 9542, 198, 15666, 25, 318 ], "slot": 47, "nopts": 2, "label_ids": [ 32, 33 ], "label_strings": [ "A", "B" ], "tokens": 48, "slot_logits": [ 14.459798812866211, 9.720666885375977 ], "p_oracle": [ 0.9900590777397156, 0.00994098000228405 ], "argmax": 0, "argmax_label": "A", "argmax_token_id": 32, "top2_margin": 0.9801180958747864 } ], "api_assembly": [ { "request_id": "r01", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "route": { "type": "choice", "choice": "repair", "confidence": 0.9706, "certainty": 0.8767, "probabilities": { "repair": 0.9706, "billing": 0.0288, "delivery": 0.0006 } }, "condition": { "type": "choice", "choice": "intact", "confidence": 0.9942, "certainty": 0.9702, "probabilities": { "missing": 0.0018, "intact": 0.9942, "wet": 0.0008, "broken": 0.0032 } }, "refund": { "type": "noul", "noul": 0.0026 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } } }, { "request_id": "r02", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "action": { "type": "choice", "choice": "close the window", "confidence": 0.9991, "certainty": 0.9932, "probabilities": { "open the window": 0.0008, "close the window": 0.9991, "turn on the lamp": 0.0001 } }, "lamp": { "type": "choice", "choice": "off", "confidence": 0.8881, "certainty": 0.67, "probabilities": { "on": 0.1094, "flashing": 0.0025, "off": 0.8881 } }, "cold": { "type": "noul", "noul": 0.9587 } }, "usage": { "input_tokens": 115, "output_tokens": 0 } } }, { "request_id": "r03", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "color": { "type": "choice", "choice": "green", "confidence": 0.9406, "certainty": 0.8258, "probabilities": { "red": 0.0111, "blue": 0.007, "yellow": 0.0251, "purple": 0.0058, "green": 0.9406, "orange": 0.0105 } }, "shape": { "type": "choice", "choice": "hexagon", "confidence": 0.9587, "certainty": 0.8801, "probabilities": { "circle": 0.004, "square": 0.0065, "triangle": 0.0082, "pentagon": 0.0097, "hexagon": 0.9587, "oval": 0.0025, "star": 0.0053, "rectangle": 0.0051 } }, "wet": { "type": "noul", "noul": 0.0035 } }, "usage": { "input_tokens": 142, "output_tokens": 0 } } }, { "request_id": "r04", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "drawer": { "type": "choice", "choice": "seven", "confidence": 0.9802, "certainty": 0.9368, "probabilities": { "one": 0.0119, "two": 0.0023, "three": 0.0012, "four": 0.0015, "five": 0.0017, "six": 0.0012, "seven": 0.9802 } }, "material": { "type": "choice", "choice": "wood", "confidence": 0.9419, "certainty": 0.8603, "probabilities": { "glass": 0.0022, "stone": 0.0075, "steel": 0.0271, "paper": 0.0028, "clay": 0.0021, "wood": 0.9419, "rubber": 0.0025, "copper": 0.0061, "fabric": 0.0014, "plastic": 0.0063 } }, "key": { "type": "noul", "noul": 0.9834 } }, "usage": { "input_tokens": 149, "output_tokens": 0 } } }, { "request_id": "r05", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "task": { "type": "choice", "choice": "watering", "confidence": 0.9951, "certainty": 0.9703, "probabilities": { "harvesting": 0.0043, "watering": 0.9951, "painting": 0.0006 } }, "leaves": { "type": "choice", "choice": "green", "confidence": 0.9961, "certainty": 0.979, "probabilities": { "brown": 0.0006, "yellow": 0.0024, "green": 0.9961, "red": 0.0009 } }, "dry": { "type": "noul", "noul": 0.9791 } }, "usage": { "input_tokens": 100, "output_tokens": 0 } } }, { "request_id": "r06", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "phase": { "type": "choice", "choice": "review", "confidence": 0.9835, "certainty": 0.9283, "probabilities": { "draft": 0.0053, "review": 0.9835, "complete": 0.0094, "cancelled": 0.0017 } }, "priority": { "type": "choice", "choice": "low", "confidence": 0.9299, "certainty": 0.7469, "probabilities": { "high": 0.0077, "medium": 0.0624, "low": 0.9299 } }, "approved": { "type": "noul", "noul": 0.008 } }, "usage": { "input_tokens": 119, "output_tokens": 0 } } }, { "request_id": "r07", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "contents": { "type": "choice", "choice": "sand", "confidence": 0.9838, "certainty": 0.9274, "probabilities": { "water": 0.0044, "sand": 0.9838, "salt": 0.0059, "seeds": 0.0059 } }, "location": { "type": "choice", "choice": "indoors", "confidence": 0.9905, "certainty": 0.9593, "probabilities": { "outdoors": 0.0037, "underground": 0.0021, "indoors": 0.9905, "on a roof": 0.003, "in a pond": 0.0006 } }, "seal": { "type": "noul", "noul": 0.0043 } }, "usage": { "input_tokens": 118, "output_tokens": 0 } } }, { "request_id": "r08", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "direction": { "type": "choice", "choice": "west", "confidence": 0.8213, "certainty": 0.5893, "probabilities": { "north": 0.0261, "south": 0.0701, "east": 0.0798, "west": 0.8213, "up": 0.0027 } }, "surface": { "type": "choice", "choice": "gravel", "confidence": 0.9964, "certainty": 0.9765, "probabilities": { "gravel": 0.9964, "ice": 0.0009, "asphalt": 0.0027 } }, "open": { "type": "noul", "noul": 0.0074 } }, "usage": { "input_tokens": 121, "output_tokens": 0 } } }, { "request_id": "r09", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "liquid": { "type": "choice", "choice": "water", "confidence": 0.9842, "certainty": 0.9162, "probabilities": { "oil": 0.0064, "water": 0.9842, "ink": 0.0094 } }, "valve": { "type": "choice", "choice": "closed", "confidence": 0.9528, "certainty": 0.8204, "probabilities": { "open": 0.0456, "closed": 0.9528, "missing": 0.0016 } }, "clarity": { "type": "choice", "choice": "clean", "confidence": 0.9975, "certainty": 0.9855, "probabilities": { "dirty": 0.0014, "cloudy": 0.0007, "frozen": 0.0004, "clean": 0.9975 } }, "fill": { "type": "score", "score": 1.96, "confidence": 0.6913, "certainty": 0.4656, "legend": { "0": "empty", "1": "one quarter full", "2": "half full", "3": "three quarters full", "4": "completely full" }, "probabilities": { "0": 0.0026, "1": 0.1749, "2": 0.6913, "3": 0.1272, "4": 0.004 }, "level_fit": { "0": 0.0036, "1": 0.2451, "2": 0.9685, "3": 0.1782, "4": 0.0056 }, "fit_mass": 1.401 } }, "usage": { "input_tokens": 288, "output_tokens": 0 } } }, { "request_id": "r10", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "surface": { "type": "choice", "choice": "smooth", "confidence": 0.9929, "certainty": 0.9671, "probabilities": { "rough": 0.0062, "smooth": 0.9929, "cracked": 0.0007, "sticky": 0.0003 } }, "color": { "type": "choice", "choice": "white", "confidence": 0.9705, "certainty": 0.8664, "probabilities": { "black": 0.0053, "white": 0.9705, "gray": 0.0241 } }, "count": { "type": "choice", "choice": "four", "confidence": 0.9366, "certainty": 0.7517, "probabilities": { "two": 0.0166, "four": 0.9366, "six": 0.0468 } }, "damage": { "type": "score", "score": 0.02, "confidence": 0.9932, "certainty": 0.9691, "legend": { "0": "no damage", "1": "minor scratches", "2": "several shallow dents", "3": "large cracks", "4": "completely broken" }, "probabilities": { "0": 0.9932, "1": 0.0022, "2": 0.0013, "3": 0.0015, "4": 0.0018 }, "level_fit": { "0": 0.9932, "1": 0.0022, "2": 0.0013, "3": 0.0015, "4": 0.0018 }, "fit_mass": 0.9999 } }, "usage": { "input_tokens": 289, "output_tokens": 0 } } }, { "request_id": "r11", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "bin": { "type": "choice", "choice": "violet", "confidence": 0.994, "certainty": 0.9793, "probabilities": { "red": 0.0009, "orange": 0.0005, "yellow": 0.0004, "green": 0.0005, "blue": 0.001, "indigo": 0.0011, "violet": 0.994, "black": 0.0004, "white": 0.0004, "gray": 0.0003, "brown": 0.0003 } } }, "usage": { "input_tokens": 87, "output_tokens": 0 } } }, { "request_id": "r12", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "slot": { "type": "choice", "choice": "173", "confidence": 0.7887, "certainty": 0.8101, "probabilities": { "1": 0.0003, "2": 0.0, "3": 0.0003, "4": 0.0022, "5": 0.0, "6": 0.0, "7": 0.0101, "8": 0.0, "9": 0.0001, "10": 0.0, "11": 0.0001, "12": 0.0001, "13": 0.0003, "14": 0.0002, "15": 0.0003, "16": 0.0005, "17": 0.1025, "18": 0.0006, "19": 0.0001, "20": 0.0, "21": 0.0, "22": 0.0, "23": 0.0, "24": 0.0001, "25": 0.0, "26": 0.0, "27": 0.0, "28": 0.0, "29": 0.0, "30": 0.001, "31": 0.0, "32": 0.0, "33": 0.0004, "34": 0.0, "35": 0.0, "36": 0.0, "37": 0.0, "38": 0.0, "39": 0.0, "40": 0.0, "41": 0.0001, "42": 0.0, "43": 0.0013, "44": 0.0, "45": 0.0, "46": 0.0, "47": 0.0, "48": 0.0, "49": 0.0, "50": 0.0, "51": 0.0, "52": 0.0, "53": 0.0, "54": 0.0, "55": 0.0, "56": 0.0019, "57": 0.0, "58": 0.0, "59": 0.0003, "60": 0.0, "61": 0.0, "62": 0.0, "63": 0.0, "64": 0.0, "65": 0.0, "66": 0.0, "67": 0.0, "68": 0.0, "69": 0.0, "70": 0.0, "71": 0.0, "72": 0.0, "73": 0.0, "74": 0.0, "75": 0.0, "76": 0.0, "77": 0.0, "78": 0.0, "79": 0.0, "80": 0.0045, "81": 0.0, "82": 0.0, "83": 0.0009, "84": 0.0, "85": 0.0, "86": 0.0, "87": 0.0, "88": 0.0, "89": 0.0, "90": 0.0001, "91": 0.0, "92": 0.0, "93": 0.0, "94": 0.0, "95": 0.0, "96": 0.0, "97": 0.0, "98": 0.0, "99": 0.0, "100": 0.0005, "101": 0.0008, "102": 0.0003, "103": 0.0005, "104": 0.0005, "105": 0.0001, "106": 0.0089, "107": 0.0001, "108": 0.0001, "109": 0.0001, "110": 0.0002, "111": 0.0004, "112": 0.0003, "113": 0.0001, "114": 0.001, "115": 0.0006, "116": 0.0002, "117": 0.0011, "118": 0.0001, "119": 0.0003, "120": 0.0, "121": 0.0001, "122": 0.0003, "123": 0.0001, "124": 0.0, "125": 0.0, "126": 0.0, "127": 0.0009, "128": 0.0, "129": 0.0, "130": 0.0002, "131": 0.0, "132": 0.0, "133": 0.0, "134": 0.0, "135": 0.0, "136": 0.0, "137": 0.0001, "138": 0.0, "139": 0.0004, "140": 0.0, "141": 0.0, "142": 0.0, "143": 0.0, "144": 0.0, "145": 0.0, "146": 0.0, "147": 0.0, "148": 0.0, "149": 0.0, "150": 0.0, "151": 0.0005, "152": 0.0, "153": 0.0, "154": 0.0002, "155": 0.0, "156": 0.0, "157": 0.0, "158": 0.0, "159": 0.0, "160": 0.0, "161": 0.0, "162": 0.0, "163": 0.0, "164": 0.0, "165": 0.0, "166": 0.0, "167": 0.0, "168": 0.0, "169": 0.0, "170": 0.0093, "171": 0.0044, "172": 0.0118, "173": 0.7887, "174": 0.0068, "175": 0.0008, "176": 0.0028, "177": 0.0005, "178": 0.001, "179": 0.0018, "180": 0.0032, "181": 0.0013, "182": 0.0026, "183": 0.0021, "184": 0.0016, "185": 0.0038, "186": 0.0009, "187": 0.0007, "188": 0.001, "189": 0.0012, "190": 0.0006, "191": 0.0002, "192": 0.0, "193": 0.0, "194": 0.0, "195": 0.0013, "196": 0.0, "197": 0.0, "198": 0.0001, "199": 0.0, "200": 0.0, "201": 0.0, "202": 0.0, "203": 0.0, "204": 0.0, "205": 0.0, "206": 0.0, "207": 0.0012, "208": 0.0, "209": 0.0, "210": 0.0, "211": 0.0, "212": 0.0, "213": 0.0, "214": 0.0, "215": 0.0, "216": 0.0, "217": 0.0, "218": 0.0, "219": 0.0, "220": 0.0007, "221": 0.0, "222": 0.0, "223": 0.0, "224": 0.0, "225": 0.0, "226": 0.0, "227": 0.0, "228": 0.0, "229": 0.0, "230": 0.0, "231": 0.0, "232": 0.0, "233": 0.0002, "234": 0.0, "235": 0.0, "236": 0.0, "237": 0.0, "238": 0.0, "239": 0.0, "240": 0.0, "241": 0.0, "242": 0.0, "243": 0.0, "244": 0.0, "245": 0.0006, "246": 0.0, "247": 0.0, "248": 0.0, "249": 0.0, "250": 0.0, "251": 0.0, "252": 0.0, "253": 0.0, "254": 0.0, "255": 0.0 } } }, "usage": { "input_tokens": 1965, "output_tokens": 0 } } }, { "request_id": "r13", "exact_equal": true, "system_one": { "model": "decider-0.8b-v1", "answers": { "status": { "type": "choice", "choice": "paused", "confidence": 0.9976, "certainty": 0.9833, "probabilities": { "running": 0.0015, "paused": 0.9976, "finished": 0.0008 } }, "restart": { "type": "noul", "noul": 0.0099 } }, "usage": { "input_tokens": 73, "output_tokens": 0 } } } ], "summary": { "requests": 13, "rows": 44, "min_top2_margin": 0.5097886919975281, "max_tokens": 1965, "wall_seconds": 252.27061412506737 } }