>>109854982
12t/s achieved with the original quant at full context + vision (RTX 3090)
commit="37b3a9e0ccba261d1cc245a971deae0b18c201ab" && \
model_folder="/mnt/AI/LLM/Qwen3.8-Flash-Next-GSQ-RCO-GGUF/" && \
model_basename="Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002" && \
model=$model_folder$model_basename'.gguf' && \
CUDA_VISIBLE_DEVICES=0 numactl --physcpubind=24-31 --membind=1 \
"$HOME/LLAMA_CPP/$commit/llama.cpp/build/bin/llama-server" \
--model "$model" \
--threads $(lscpu | grep "Core(s) per socket" | awk '{print $4}') \
--threads-batch $(lscpu | grep "Core(s) per socket" | awk '{print $4}') \
--temp 1.0 --top-p 0.95 --top-k 20 --min-p 0.0 \
--no-warmup \
--port 8001 \
--host 0.0.0.0 \
--flash-attn on \
--fit on --fit-target 512 \
--reasoning-format deepseek \
-ctk q8_0 -ctv q8_0 \
--jinja \
-b 4096 \
-ub 1024 \
--metrics \
--chat-template-kwargs '{"reasoning_effort":"xhigh"}' \
--lazy-mode on \
--parallel 1 \
--mmproj $model_folder'mmproj-Qwen3.8-Flash-Next-BF16.gguf' \
--ctx-size 0
Thank you, anon, for hinting on --fit on
option
