{ "artifacts": [ { "file": "Qwen3.5-2B-VL_int8.litertlm", "sha256": "f8821e403dcfc939a033f6b2c9633a5d6b78c29a380ca825e0b90a62df61afe5", "size_mb": 3003.03 } ], "benchmarks": [], "conversion": { "command": "TODO (qwen35_work/ in hf-to-litertlm; invocation not stated on the card)", "quantization": "text decoder: int8 dynamic on linears + embedding (convs and the delta rule stay float), fp32 activations declared; vision: the checkpoint's own ViT shipped fp16 (int8 drops the tower's correlation to 0.97-0.98 on real photographs), int8 adapter; static 512x512; six-signature prefill ladder (HF card file table + Honest notes)", "tool": "litert-torch plus a hybrid-cache patch (reproduction script + patch: hf-to-litertlm qwen35_work/) — rank-4 chunk kernel for the gated delta rule, export cache layers for the linear-attention layers, re-authored static 512x512 vision tower, pad guard on the externalised-embedder graph (HF card Conversion notes)", "tool_version": "TODO (the HF card names the converter but not its version; no export log in the sources)" }, "cross_runtime": [], "delegation": null, "device": { "records": [ { "device": "galaxy-s26", "run": { "accelerator": "cpu", "context_length": null, "date": "2026-09-05", "decode_tokens_per_s": 18.98, "delegated_ops": 24542, "env": { "device": "Galaxy S26 (SM-S942Q)", "machine_label": "galaxy-s26-cold-cache-cooled", "os_build": "Android 16", "runtime": "litert-lm", "runtime_version": "0.16.0", "soc": "Qualcomm SM8850", "vendor_sdk": null }, "error": null, "evidence": [ "VERBOSE: Replacing 24542 out of 25703 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 2245 partitions for subgraph 0 (prefill_1024).", "VERBOSE: Replacing 17648 out of 18791 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 2209 partitions for subgraph 1 (prefill_256).", "VERBOSE: Replacing 15632 out of 16775 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 2209 partitions for subgraph 2 (prefill_64).", "VERBOSE: Replacing 15722 out of 16865 node(s) with delegate (TfLiteXNNPackDelegate) node, yielding 2209 partitions for subgraph 3 (prefill_16).", "results block: prefill=180.67 decode=18.98 tokens/s" ], "failure_class": null, "full_delegation": false, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 3889.0, "init_s": 16.97616, "prefill_tokens": 207.0 }, "output_match": null, "peak_mem_mb": null, "prefill_tokens_per_s": 180.67, "provenance": "measured", "runs": true, "total_ops": 25703, "ttft_ms": 1200.0 }, "source": "data/device_runs/0.16.0/2026-09-05/qwen35-2b-vl-int8__galaxy-s26.json" }, { "device": "galaxy-s26", "run": { "accelerator": "gpu", "context_length": null, "date": "2026-09-05", "decode_tokens_per_s": 17.47, "delegated_ops": 25703, "env": { "device": "Galaxy S26 (SM-S942Q)", "machine_label": "galaxy-s26-cold-cache-cooled", "os_build": "Android 16", "runtime": "litert-lm", "runtime_version": "0.16.0", "soc": "Qualcomm SM8850", "vendor_sdk": null }, "error": null, "evidence": [ "VERBOSE: Replacing 25703 out of 25703 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 0 (prefill_1024).", "VERBOSE: Replacing 18791 out of 18791 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 1 (prefill_256).", "VERBOSE: Replacing 16775 out of 16775 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 2 (prefill_64).", "VERBOSE: Replacing 16865 out of 16865 node(s) with delegate (LITERT_CL) node, yielding 1 partitions for subgraph 3 (prefill_16).", "results block: prefill=450.06 decode=17.47 tokens/s" ], "failure_class": null, "full_delegation": true, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 386.0, "init_s": 76.72917, "prefill_tokens": 207.0 }, "output_match": null, "peak_mem_mb": null, "prefill_tokens_per_s": 450.06, "provenance": "measured", "runs": true, "total_ops": 25703, "ttft_ms": 520.0 }, "source": "data/device_runs/0.16.0/2026-09-05/qwen35-2b-vl-int8__galaxy-s26.json" }, { "device": "raspberry-pi-5", "run": { "accelerator": "cpu", "context_length": null, "date": "2026-09-01", "decode_tokens_per_s": 4.24, "delegated_ops": null, "env": { "device": "Raspberry Pi 5 Model B Rev 1.1", "machine_label": "raspberry-pi-5-cooled-52c", "os_build": "Linux-6.18.34+rpt-rpi-2712-aarch64-with-glibc2.41", "runtime": "litert-lm", "runtime_version": "0.16.1", "soc": "Broadcom BCM2712", "vendor_sdk": null }, "error": null, "evidence": [ "pi5 LLM sweep row: `litert-lm benchmark --backend cpu --cpu-thread-count 4 -p 256 -d 256 --runs 1 --cache memory` (wave-2 driver pi5_llm_bench.py; --cache memory rather than the house --cache no, which OOM-kills every >=1.2B file on the 8 GB Pi — equivalence measured on granite-350m int8, +2-3%), 3 invocations per file with cool-down to <=52 C between them, vcgencmd measure_temp + get_throttled logged per invocation, peak RSS polled from /proc; throughput = median of the three invocations (spread in metrics); a row counts as measured only when the real-generation gate (`litert-lm run`, degenerate-output check) passed and every invocation exited 0 with get_throttled 0x0", "versions: cpu=Raspberry Pi 5 Model B Rev 1.1, litert-lm=0.16.1, litert-lm-api=0.16.1, platform=Linux-6.18.34+rpt-rpi-2712-aarch64-with-glibc2.41, python=3.13.5", "cache mode 'memory'; -p 256 -d 256 --runs 1 --cpu-thread-count 4", "gate ('What is 17 plus 26? Answer with the number only.'): status pass, exit 0, wall 142.9 s, output head '43'", "invocation 0: exit=0 wall_s=256.1 temp 47.2->56.5C throttled=0x0 prefill_tps=55.91 decode_tps=4.2 ttft_s=5.0085 init_s=114.455 peak_rss_mb=3267", "invocation 1: exit=0 wall_s=258.1 temp 49.9->54.3C throttled=0x0 prefill_tps=55.4 decode_tps=4.24 ttft_s=5.0444 init_s=115.6445 peak_rss_mb=3268", "invocation 2: exit=0 wall_s=258.1 temp 50.5->52.7C throttled=0x0 prefill_tps=56.31 decode_tps=4.28 ttft_s=5.0379 init_s=115.8296 peak_rss_mb=3268" ], "failure_class": null, "full_delegation": null, "latency_p50_ms": null, "loads": true, "max_abs_diff": null, "max_rel_diff": null, "metrics": { "decode_tokens": 256.0, "decode_tps_max": 4.28, "decode_tps_min": 4.2, "init_s": 115.6445, "init_s_max": 115.8296, "init_s_min": 114.455, "invocations": 3.0, "prefill_tokens": 256.0, "prefill_tps_max": 56.31, "prefill_tps_min": 55.4, "runs_per_invocation": 1.0, "threads": 4.0, "ttft_s_max": 5.0444, "ttft_s_min": 5.0085 }, "output_match": null, "peak_mem_mb": 3268.0, "prefill_tokens_per_s": 55.91, "provenance": "measured", "runs": true, "total_ops": null, "ttft_ms": 5037.9 }, "source": "data/device_runs/0.16.1/2026-09-01/qwen35-2b-vl-int8__raspberry-pi-5.json" } ] }, "model": { "family": "qwen3_5", "id": "qwen35-2b-vl-int8", "license": "apache-2.0", "source_url": "https://huggingface.co/litert-community/Qwen3.5-2B", "task": "image-text-to-text" }, "pitfalls": [ "Six prefill signatures, not eleven: every exported signature is charged engine memory whether or not it is called; the text build's full 1-1024 ladder peaks ~5.3 GB on iPhone GPU and fits, but adding the vision tower pushes the same ladder past a 12 GB phone's jetsam ceiling (HF card Honest notes).", "Positions are 1-D: the fast_vlm contract feeds sequential positions, so the checkpoint's M-RoPE collapses to plain RoPE; against the full M-RoPE reference on nine image/prompt pairs one generation is identical and the other eight are fluent same-content paraphrases (HF card Honest notes).", "Android is not gated for this build: the text build's notes apply (Pixel-class 8 GB GPU does not fit), and on Mali the fp16 vision encoder is known to crash the device on other models of this shape (HF card Honest notes); this repo's S26 GPU/CPU and Pi 5 CPU rows come from the S7 backfill / Pi 5 wave 2 (device runs).", "Composite 8-question probe: on Mac this file answers 7 of 8, missing only 17 + 25 identically on GPU and CPU (the model, not the conversion; the text build records the same slip) (HF card Honest notes).", "The vision tower is re-authored for one static 512x512 image (Conv3d patch-embed folded to Conv2d over the duplicated temporal frame, learned position embeddings resampled) because the upstream dynamic-resolution grid_thw preprocessing aborts torch.export (HF card Conversion notes).", "LiteRT-LM .litertlm bundle: LiteRT.js cannot run it, so delegation stays null and there is no browser block; device rows come from data/device_runs/ (Pi 5 / S26 / Pixel 8a / Mac / iPhone as measured)." ], "schema_version": "1.2" }