{ "artifacts": [ { "file": "OLMo-2-1B-Instruct_q4_block32_ekv4096.litertlm", "sha256": "669484d528d9b981ddf5057126f3b0e629e2fe793310f95959f8463eb538b811", "size_mb": 888.101 } ], "benchmarks": [], "conversion": { "command": "TODO", "quantization": "int4 weights — blockwise (block 32) + OCTAV optimal clipping, symmetric; embedding INT8; integer compute", "tool": "litert-torch export_hf (official upstream main, no fork patches)", "tool_version": "TODO" }, "cross_runtime": [], "delegation": null, "device": { "records": [ { "device": "galaxy-s26", "run": { "accelerator": "cpu", "context_length": null, "date": "2026-09-05", "decode_tokens_per_s": 12.1, "delegated_ops": 1031, "env": { "device": "Galaxy S26 (SM-S942Q)", "machine_label": "galaxy-s26-cold-cache-cooled", "os_build": "Android 16", "runtime": "litert-lm", "runtime_version": "0.16.0", "soc": "Qualcomm SM8850", "vendor_sdk": null }, "error": null, "evidence": [ "VERBOSE: Replacing 604 out of 705 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 125 partitions for subgraph 0 (prefill_128).", "VERBOSE: Replacing 566 out of 670 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 132 partitions for subgraph 1 (decode).", "VERBOSE: Replacing 7 out of 7 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 1 partitions for subgraph 2 (odml.rms_norm.impl).", "VERBOSE: Replacing 7 out of 7 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 1 partitions for subgraph 3 (odml.rms_norm.impl_0).", "results block: prefill=168.67 decode=12.1 tokens/s" ], "failure_class": null, "full_delegation": false, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 182.0, "init_s": 1.3572, "prefill_tokens": 205.0 }, "output_match": null, "peak_mem_mb": null, "prefill_tokens_per_s": 168.67, "provenance": "measured", "runs": true, "total_ops": 1071, "ttft_ms": 1300.0 }, "source": "data/device_runs/0.16.0/2026-09-05/olmo-2-1b-instruct-q4-block32-ekv4096__galaxy-s26.json" }, { "device": "galaxy-s26", "run": { "accelerator": "gpu", "context_length": null, "date": "2026-08-24", "decode_tokens_per_s": 22.97, "delegated_ops": 705, "env": { "device": "Galaxy S26 (SM-S942Q)", "machine_label": "galaxy-s26-cl-pinned", "os_build": "Android 16", "runtime": "litert-lm", "runtime_version": "0.16.0", "soc": "Qualcomm SM8850", "vendor_sdk": null }, "error": null, "evidence": [ "VERBOSE: Replacing 705 out of 705 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 0 (prefill_128).", "VERBOSE: Replacing 670 out of 670 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 1 (decode).", "results block: prefill=494.66 decode=22.97 tokens/s" ], "failure_class": null, "full_delegation": true, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 320.0, "init_s": 6.56675, "prefill_tokens": 210.0 }, "output_match": null, "peak_mem_mb": null, "prefill_tokens_per_s": 494.66, "provenance": "measured", "runs": true, "total_ops": 705, "ttft_ms": 470.0 }, "source": "data/device_runs/0.16.0/2026-08-24/olmo-2-1b-instruct-q4-block32-ekv4096__galaxy-s26.json" }, { "device": "raspberry-pi-5", "run": { "accelerator": "cpu", "context_length": null, "date": "2026-09-01", "decode_tokens_per_s": 2.13, "delegated_ops": null, "env": { "device": "Raspberry Pi 5 Model B Rev 1.1", "machine_label": "raspberry-pi-5-cooled-52c", "os_build": "Linux-6.18.34+rpt-rpi-2712-aarch64-with-glibc2.41", "runtime": "litert-lm", "runtime_version": "0.16.1", "soc": "Broadcom BCM2712", "vendor_sdk": null }, "error": null, "evidence": [ "pi5 LLM sweep row: `litert-lm benchmark --backend cpu --cpu-thread-count 4 -p 256 -d 256 --runs 1 --cache memory` (wave-2 driver pi5_llm_bench.py; --cache memory rather than the house --cache no, which OOM-kills every >=1.2B file on the 8 GB Pi — equivalence measured on granite-350m int8, +2-3%), 3 invocations per file with cool-down to <=52 C between them, vcgencmd measure_temp + get_throttled logged per invocation, peak RSS polled from /proc; throughput = median of the three invocations (spread in metrics); a row counts as measured only when the real-generation gate (`litert-lm run`, degenerate-output check) passed and every invocation exited 0 with get_throttled 0x0", "versions: cpu=Raspberry Pi 5 Model B Rev 1.1, litert-lm=0.16.1, litert-lm-api=0.16.1, platform=Linux-6.18.34+rpt-rpi-2712-aarch64-with-glibc2.41, python=3.13.5", "cache mode 'memory'; -p 256 -d 256 --runs 1 --cpu-thread-count 4", "gate ('What is 17 plus 26? Answer with the number only.'): status pass, exit 0, wall 29.2 s, output head '43'", "invocation 0: exit=0 wall_s=296.2 temp 46.1->53.2C throttled=0x0 prefill_tps=40.32 decode_tps=2.13 ttft_s=7.6683 init_s=20.8993 peak_rss_mb=2326", "invocation 1: exit=0 wall_s=290.2 temp 50.5->54.3C throttled=0x0 prefill_tps=39.96 decode_tps=2.13 ttft_s=7.628 init_s=20.601 peak_rss_mb=2325", "invocation 2: exit=0 wall_s=292.2 temp 49.9->52.7C throttled=0x0 prefill_tps=39.44 decode_tps=2.12 ttft_s=7.8113 init_s=21.0324 peak_rss_mb=2325" ], "failure_class": null, "full_delegation": null, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 256.0, "decode_tps_max": 2.13, "decode_tps_min": 2.12, "init_s": 20.8993, "init_s_max": 21.0324, "init_s_min": 20.601, "invocations": 3.0, "prefill_tokens": 256.0, "prefill_tps_max": 40.32, "prefill_tps_min": 39.44, "runs_per_invocation": 1.0, "threads": 4.0, "ttft_s_max": 7.8113, "ttft_s_min": 7.628 }, "output_match": null, "peak_mem_mb": 2326.0, "prefill_tokens_per_s": 39.96, "provenance": "measured", "runs": true, "total_ops": null, "ttft_ms": 7668.3 }, "source": "data/device_runs/0.16.1/2026-09-01/olmo-2-1b-instruct-q4-block32-ekv4096__raspberry-pi-5.json" } ] }, "model": { "family": "olmo2", "id": "olmo-2-1b-instruct-q4-block32-ekv4096", "license": "apache-2.0", "source_url": "https://huggingface.co/allenai/OLMo-2-0425-1B-Instruct", "task": "text-generation" }, "pitfalls": [ "GSM8K (n=100, greedy, 0-shot CoT): int4 63.0% vs bf16 72.0% — at 1B, 4-bit costs ~9 pt because a small model has less redundancy to absorb int4 rounding than a 3B+ (card Quality section).", "An int8 build recovers only ~2 pt (65%) for +60% size — int4 is shipped as the size/quality trade-off (card Quality section)." ], "schema_version": "1.2" }