metal : try to improve batched decoding

ggerganov · Dec 1, 2023 · 3cb1c34 · 3cb1c34
1 parent 3e68df8
commit 3cb1c34
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/ggml-metal.m b/ggml-metal.m
@@ -1083,7 +1083,7 @@ void ggml_metal_graph_compute(
 
                             // find the break-even point where the matrix-matrix kernel becomes more efficient compared
                             // to the matrix-vector kernel
-                            int ne11_mm_min = 1;
+                            int ne11_mm_min = src0t == GGML_TYPE_F16 ? 1 : 16;
 
 #if 0
                             // the numbers below are measured on M2 Ultra for 7B and 13B models