{ "artifacts": [ { "file": "model.litertlm", "sha256": "12a5a5089880044f8cebba1e4f5380efbfe5e55461f86cf6878d20dbb9234d40", "size_mb": 1961.809 } ], "benchmarks": [], "conversion": { "command": "CACHE=4096 EXTERNALIZE_EMBEDDER=1 export_simple_template.py WeiboAI/VibeThinker-3B templates/chatml_simple.jinja BOCTAV4", "quantization": "int4 weights, block 32, symmetric + OCTAV optimal clipping; embeddings INT8 (externalized section)", "tool": "litert-torch (hf-to-litertlm reproduce_llm.sh key `vibethinker-3b`)", "tool_version": "TODO" }, "cross_runtime": [], "delegation": null, "device": { "records": [ { "device": "galaxy-s26", "run": { "accelerator": "gpu", "context_length": null, "date": "2026-08-24", "decode_tokens_per_s": 13.88, "delegated_ops": 1603, "env": { "device": "Galaxy S26 (SM-S942Q)", "machine_label": "galaxy-s26-cl-pinned", "os_build": "Android 16", "runtime": "litert-lm", "runtime_version": "0.16.0", "soc": "Qualcomm SM8850", "vendor_sdk": null }, "error": null, "evidence": [ "VERBOSE: Replacing 1603 out of 1603 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 0 (prefill_128).", "VERBOSE: Replacing 1452 out of 1452 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 1 (decode).", "results block: prefill=157.44 decode=13.88 tokens/s" ], "failure_class": null, "full_delegation": true, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 3894.0, "init_s": 18.8138, "prefill_tokens": 202.0 }, "output_match": null, "peak_mem_mb": null, "prefill_tokens_per_s": 157.44, "provenance": "measured", "runs": true, "total_ops": 1603, "ttft_ms": 1360.0 }, "source": "data/device_runs/0.16.0/2026-08-24/vibethinker-3b__galaxy-s26.json" }, { "device": "mac-studio-m4-max", "run": { "accelerator": "gpu", "context_length": null, "date": "2026-07-23", "decode_tokens_per_s": 93.16, "delegated_ops": null, "env": { "device": "Mac Studio (M4 Max)", "machine_label": "mac-studio-m4-max", "os_build": null, "runtime": "litert-lm", "runtime_version": "0.14.0", "soc": "Apple M4 Max", "vendor_sdk": null }, "error": null, "evidence": [ "results block: prefill=1358.96 decode=93.16 tokens/s, init=2.8258 s" ], "failure_class": null, "full_delegation": null, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 256.0, "init_s": 2.8258, "max_num_tokens": 1024.0, "prefill_tokens": 256.0 }, "output_match": null, "peak_mem_mb": null, "prefill_tokens_per_s": 1358.96, "provenance": "measured", "runs": true, "total_ops": null, "ttft_ms": 199.1 }, "source": "data/device_runs/0.14.0/2026-07-23/vibethinker-3b__mac-studio-m4-max.json" }, { "device": "pixel-8a", "run": { "accelerator": "gpu", "context_length": null, "date": "2026-08-17", "decode_tokens_per_s": 7.58, "delegated_ops": 1603, "env": { "device": "Pixel 8a", "machine_label": "pixel-8a", "os_build": null, "runtime": "litert-lm", "runtime_version": "0.16.0", "soc": "Tensor G3", "vendor_sdk": null }, "error": null, "evidence": [ "VERBOSE: Replacing 1603 out of 1603 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 0 (prefill_128).", "VERBOSE: Replacing 1452 out of 1452 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 1 (decode).", "results block: prefill=12.6 decode=7.58 tokens/s" ], "failure_class": null, "full_delegation": true, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 4079.0, "init_s": 23.10211, "prefill_tokens": 17.0 }, "output_match": null, "peak_mem_mb": null, "prefill_tokens_per_s": 12.6, "provenance": "measured", "runs": true, "total_ops": 1603, "ttft_ms": 1480.0 }, "source": "data/device_runs/0.16.0/2026-08-17/vibethinker-3b__pixel-8a.json" }, { "device": "raspberry-pi-5", "run": { "accelerator": "cpu", "context_length": null, "date": "2026-09-01", "decode_tokens_per_s": 3.36, "delegated_ops": null, "env": { "device": "Raspberry Pi 5 Model B Rev 1.1", "machine_label": "raspberry-pi-5-cooled-52c", "os_build": "Linux-6.18.34+rpt-rpi-2712-aarch64-with-glibc2.41", "runtime": "litert-lm", "runtime_version": "0.16.1", "soc": "Broadcom BCM2712", "vendor_sdk": null }, "error": null, "evidence": [ "pi5 LLM sweep row: `litert-lm benchmark --backend cpu --cpu-thread-count 4 -p 256 -d 256 --runs 1 --cache memory` (wave-2 driver pi5_llm_bench.py; --cache memory rather than the house --cache no, which OOM-kills every >=1.2B file on the 8 GB Pi — equivalence measured on granite-350m int8, +2-3%), 3 invocations per file with cool-down to <=52 C between them, vcgencmd measure_temp + get_throttled logged per invocation, peak RSS polled from /proc; throughput = median of the three invocations (spread in metrics); a row counts as measured only when the real-generation gate (`litert-lm run`, degenerate-output check) passed and every invocation exited 0 with get_throttled 0x0", "versions: cpu=Raspberry Pi 5 Model B Rev 1.1, litert-lm=0.16.1, litert-lm-api=0.16.1, platform=Linux-6.18.34+rpt-rpi-2712-aarch64-with-glibc2.41, python=3.13.5", "cache mode 'memory'; -p 256 -d 256 --runs 1 --cpu-thread-count 4", "gate ('What is 17 plus 26? Answer with the number only.'): status pass, exit 0, wall 106.6 s, output head '[thought] The user asks: \"What is 17 plus 26? Answer with the number only.\" So t'", "invocation 0: exit=0 wall_s=252.1 temp 49.4->52.1C throttled=0x0 prefill_tps=22.53 decode_tps=3.36 ttft_s=13.6183 init_s=48.4717 peak_rss_mb=2808", "invocation 1: exit=0 wall_s=248.1 temp 50.5->54.9C throttled=0x0 prefill_tps=23.05 decode_tps=3.41 ttft_s=13.3687 init_s=48.3648 peak_rss_mb=2808", "invocation 2: exit=0 wall_s=252.1 temp 51.0->52.7C throttled=0x0 prefill_tps=22.41 decode_tps=3.36 ttft_s=13.6874 init_s=48.4021 peak_rss_mb=2799" ], "failure_class": null, "full_delegation": null, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 256.0, "decode_tps_max": 3.41, "decode_tps_min": 3.36, "init_s": 48.4021, "init_s_max": 48.4717, "init_s_min": 48.3648, "invocations": 3.0, "prefill_tokens": 256.0, "prefill_tps_max": 23.05, "prefill_tps_min": 22.41, "runs_per_invocation": 1.0, "threads": 4.0, "ttft_s_max": 13.6874, "ttft_s_min": 13.3687 }, "output_match": null, "peak_mem_mb": 2808.0, "prefill_tokens_per_s": 22.53, "provenance": "measured", "runs": true, "total_ops": null, "ttft_ms": 13618.3 }, "source": "data/device_runs/0.16.1/2026-09-01/vibethinker-3b__raspberry-pi-5.json" } ] }, "model": { "family": "qwen2", "id": "vibethinker-3b", "license": "mit", "source_url": "https://huggingface.co/litert-community/VibeThinker-3B", "task": "text-generation" }, "pitfalls": [ "Block 32 ONLY. This is a precision-sensitive math model: block-128 int4 (a quarter of the dequant scales) collapsed to 64% on GSM8K, −33 points, while block 32 holds at 90%. Only the block-32 build is published — and the opposite holds for general-purpose 4B reasoning models (HF card).", "Math-specialised: on general-knowledge prompts it reasons at length and can settle on a wrong answer. Measured 2026-08-17, its CPU and GPU legs are wrong in the same shape with different hallucinated content, which is how the behaviour is attributable to the model rather than the backend (CATALOG_REGATE.md).", "Measured on an 8 GB phone 2026-08-17: fully delegated on Mali OpenCL — 1603/1603 prefill nodes, 1452/1452 decode, zero rejections. Neither leg printed BenchmarkInfo because the model generated to the token cap, so speed figures for this model need a prompt it will actually finish." ], "schema_version": "1.2" }