{ "artifacts": [ { "file": "model.litertlm", "sha256": "4b4adfe6fd794451116bca61db58f0e2fb9fc3542da8d04d11ba4635693f4533", "size_mb": 2459.791 } ], "benchmarks": [], "conversion": { "command": "bash scripts/reproduce_llm.sh nanbeige4.2-3b", "quantization": "int4 blockwise-32 + OCTAV clipping, symmetric; embedding int8", "tool": "hf-to-litertlm", "tool_version": "hf-to-litertlm a59a492 (2026-08-01 public mirror of the conversion-session state)" }, "cross_runtime": [], "delegation": null, "device": { "records": [ { "device": "galaxy-s26", "run": { "accelerator": "cpu", "context_length": null, "date": "2026-09-05", "decode_tokens_per_s": 4.1, "delegated_ops": 2397, "env": { "device": "Galaxy S26 (SM-S942Q)", "machine_label": "galaxy-s26-cold-cache-cooled", "os_build": "Android 16", "runtime": "litert-lm", "runtime_version": "0.16.0", "soc": "Qualcomm SM8850", "vendor_sdk": null }, "error": null, "evidence": [ "VERBOSE: Replacing 2397 out of 2492 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 89 partitions for subgraph 0 (prefill_128).", "VERBOSE: Replacing 2130 out of 2224 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 90 partitions for subgraph 1 (decode).", "VERBOSE: Replacing 1 out of 4 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 2 partitions for subgraph 0 (decode_embedder).", "VERBOSE: Replacing 1 out of 4 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 2 partitions for subgraph 1 (prefill_embedder_128).", "results block: prefill=20.92 decode=4.1 tokens/s" ], "failure_class": null, "full_delegation": false, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 3871.0, "init_s": 9.25837, "prefill_tokens": 225.0 }, "output_match": null, "peak_mem_mb": null, "prefill_tokens_per_s": 20.92, "provenance": "measured", "runs": true, "total_ops": 2492, "ttft_ms": 11000.0 }, "source": "data/device_runs/0.16.0/2026-09-05/nanbeige4.2-3b__galaxy-s26.json" }, { "device": "galaxy-s26", "run": { "accelerator": "gpu", "context_length": null, "date": "2026-08-24", "decode_tokens_per_s": null, "delegated_ops": 2492, "env": { "device": "Galaxy S26 (SM-S942Q)", "machine_label": "galaxy-s26-cl-pinned", "os_build": "Android 16", "runtime": "litert-lm", "runtime_version": "0.16.0", "soc": "Qualcomm SM8850", "vendor_sdk": null }, "error": null, "evidence": [ "VERBOSE: Replacing 2492 out of 2492 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 0 (prefill_128).", "VERBOSE: Replacing 2224 out of 2224 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 1 (decode)." ], "failure_class": "engine_create_failed", "full_delegation": true, "latency_p50_ms": null, "loads": false, "max_abs_diff": null, "max_rel_diff": null, "output_match": null, "peak_mem_mb": null, "prefill_tokens_per_s": null, "provenance": "measured", "runs": false, "total_ops": 2492, "ttft_ms": null }, "source": "data/device_runs/0.16.0/2026-08-24/nanbeige4.2-3b__galaxy-s26.json" }, { "device": "mac-studio-m4-max", "run": { "accelerator": "cpu", "context_length": null, "date": "2026-09-06", "decode_tokens_per_s": null, "delegated_ops": null, "env": { "device": "Mac Studio (M4 Max)", "machine_label": "mac-studio-m4-max", "os_build": null, "runtime": "litert-lm", "runtime_version": "0.17.0", "soc": null, "vendor_sdk": null }, "error": null, "evidence": [ "compat_check status ok (runtime /Users/USER/code/litertlm-convert/.qa-venvs/litert-lm-0.17.0/bin/litert-lm 0.17.0); fixed-question answer: '[thought] We are asked: \"What is 17 + 25? Answer briefly.\" So we just need to compute the sum and give a brief answer.\\n\\n17 + 25 = 42. Let me verify: 17 + 20 = 37, plus 5 = 42. Yes.\\n\\nAnswer should be brief: \"42\". [/thought]\\n\\n\\n42'" ], "failure_class": null, "full_delegation": null, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "output_match": null, "peak_mem_mb": null, "prefill_tokens_per_s": null, "provenance": "measured", "runs": true, "total_ops": null, "ttft_ms": null }, "source": "data/device_runs/0.17.0/2026-09-06/nanbeige4.2-3b__mac-studio-m4-max.json" }, { "device": "raspberry-pi-5", "run": { "accelerator": "cpu", "context_length": null, "date": "2026-09-01", "decode_tokens_per_s": 1.06, "delegated_ops": null, "env": { "device": "Raspberry Pi 5 Model B Rev 1.1", "machine_label": "raspberry-pi-5-cooled-52c", "os_build": "Linux-6.18.34+rpt-rpi-2712-aarch64-with-glibc2.41", "runtime": "litert-lm", "runtime_version": "0.16.1", "soc": "Broadcom BCM2712", "vendor_sdk": null }, "error": null, "evidence": [ "pi5 LLM sweep row: `litert-lm benchmark --backend cpu --cpu-thread-count 4 -p 256 -d 256 --runs 1 --cache memory` (wave-2 driver pi5_llm_bench.py; --cache memory rather than the house --cache no, which OOM-kills every >=1.2B file on the 8 GB Pi — equivalence measured on granite-350m int8, +2-3%), 3 invocations per file with cool-down to <=52 C between them, vcgencmd measure_temp + get_throttled logged per invocation, peak RSS polled from /proc; throughput = median of the three invocations (spread in metrics); a row counts as measured only when the real-generation gate (`litert-lm run`, degenerate-output check) passed and every invocation exited 0 with get_throttled 0x0", "versions: cpu=Raspberry Pi 5 Model B Rev 1.1, litert-lm=0.16.1, litert-lm-api=0.16.1, platform=Linux-6.18.34+rpt-rpi-2712-aarch64-with-glibc2.41, python=3.13.5", "cache mode 'memory'; -p 256 -d 256 --runs 1 --cpu-thread-count 4", "gate ('What is 17 plus 26? Answer with the number only.'): status pass, exit 0, wall 167.8 s, output head '[thought] We are asked: \"What is 17 plus 26? Answer with the number only.\" So we'", "invocation 0: exit=0 wall_s=728.5 temp 51.6->52.1C throttled=0x0 prefill_tps=5.73 decode_tps=1.06 ttft_s=46.2487 init_s=65.9844 peak_rss_mb=4311", "invocation 1: exit=0 wall_s=728.5 temp 49.4->52.7C throttled=0x0 prefill_tps=5.69 decode_tps=1.08 ttft_s=45.8555 init_s=69.4478 peak_rss_mb=4280", "invocation 2: exit=0 wall_s=728.5 temp 51.0->53.2C throttled=0x0 prefill_tps=5.83 decode_tps=1.06 ttft_s=45.4005 init_s=69.6871 peak_rss_mb=4296" ], "failure_class": null, "full_delegation": null, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 256.0, "decode_tps_max": 1.08, "decode_tps_min": 1.06, "init_s": 69.4478, "init_s_max": 69.6871, "init_s_min": 65.9844, "invocations": 3.0, "prefill_tokens": 256.0, "prefill_tps_max": 5.83, "prefill_tps_min": 5.69, "runs_per_invocation": 1.0, "threads": 4.0, "ttft_s_max": 46.2487, "ttft_s_min": 45.4005 }, "output_match": null, "peak_mem_mb": 4311.0, "prefill_tokens_per_s": 5.73, "provenance": "measured", "runs": true, "total_ops": null, "ttft_ms": 45855.5 }, "source": "data/device_runs/0.16.1/2026-09-01/nanbeige4.2-3b__raspberry-pi-5.json" } ] }, "model": { "family": "nanbeige", "id": "nanbeige4.2-3b", "license": "apache-2.0", "source_url": "https://huggingface.co/Nanbeige/Nanbeige4.2-3B", "task": "text-generation" }, "pitfalls": [ "Sampling is required: the model collapses under greedy decoding — use the official recipe (temperature 0.6, top_k 20, top_p 0.95, per its generation_config.json); do not run at temperature 0.", "CPU backend only: the current GPU delegate produces incorrect output on this 44-layer unrolled graph.", "Looped transformer (22 layers x num_loops=2, unrolled at export): ~2x the per-token compute of a normal 3B — expect roughly 8B-class decode rates; 44 KV-cache slots, one per (loop, layer).", "transformers 5.x zeroes the modeling's init-computed rotary inv_freq buffer — rope must be recomputed from config at export.", "The chat template auto-opens the model's block (official template behaviour); the visible answer follows ." ], "schema_version": "1.2" }