feat: --no-vision loads a vision-capable checkpoint as text-only

README.md

@@ -629,6 +629,7 @@ curl http://localhost:5413/v1/chat/completions \
 | `--host` | `127.0.0.1` | Host to bind |
 | `--vision` | `false` | Enable VLM (vision-language model) mode for image inputs |
 | `--audio` | `false` | Enable ALM (audio-language model) mode for audio inputs |
+| `--no-vision` | `false` | Load a vision-capable checkpoint as a text-only LLM, skipping VLM auto-detection. Text-only workloads (coding agents) then get the prompt cache, which is still skipped for most VLM-loaded models (Gemma 4 text-only requests are already cached, so the flag is unnecessary there). Mutually exclusive with `--vision` |
 | `--max-tokens` | `2048` | Max tokens limit per generation |
 | `--prefill-size`| `512`  | Prompt prefill chunk size (micro-batching for long contexts) |
 | `--top-p` | `1.0` | Default top-p nucleus sampling (overridable per-request) |

Sources/SwiftLM/Server.swift

@@ -564,6 +564,9 @@ struct MLXServer: AsyncParsableCommand {
     @Flag(name: .long, help: "Enable VLM (vision-language model) mode for image inputs")
     var vision: Bool = false
 
+    @Flag(name: .long, help: "Load a vision-capable checkpoint as a text-only LLM, skipping VLM auto-detection. Text-only workloads then get the prompt cache, which is still skipped for most VLM-loaded models (Gemma 4 text-only requests are cached)")
+    var noVision: Bool = false
+
     @Flag(name: .long, help: "Enable ALM (audio-language model) mode for audio inputs")
     var audio: Bool = false
 
@@ -618,6 +621,12 @@ struct MLXServer: AsyncParsableCommand {
     @Option(name: .long, help: "Assistant checkpoint providing MTP heads, for model families that ship them separately instead of in the main checkpoint (Gemma 4). Ignored when the main model carries its own MTP heads (Qwen3.5, DeepSeek V4).")
     var mtpAssistantModel: String?
 
+    func validate() throws {
+        if vision && noVision {
+            throw ValidationError("--vision and --no-vision are mutually exclusive.")
+        }
+    }
+
     /// Entry point. Delegates to `runInner`, then — on any thrown error, at any
     /// stage — self-reports it as an `exiting` event before rethrowing, so
     /// process exit status is unchanged but the daemon gets a positive signal
@@ -978,12 +987,16 @@ struct MLXServer: AsyncParsableCommand {
         // speculative-decoding suite. --vision remains a valid explicit override.
         let speculativeDecodingRequested = self.draftModel != nil || self.dflash || self.mtp
         let autoDetectedVision = !self.audio && architecture.supportsVision
-            && !speculativeDecodingRequested
+            && !speculativeDecodingRequested && !self.noVision
         var isVision = self.vision || autoDetectedVision
         if architecture.supportsVision, !self.vision, !self.audio, speculativeDecodingRequested {
             print(
                 "[SwiftLM] Note: \(architecture.modelType ?? "unknown") reports vision support, but speculative/MTP decoding was requested; loading as a text-only LLM."
             )
+        } else if architecture.supportsVision, !self.audio, self.noVision {
+            print(
+                "[SwiftLM] Note: \(architecture.modelType ?? "unknown") reports vision support, but --no-vision was given; loading as a text-only LLM."
+            )
         } else if autoDetectedVision {
             print(
                 "[SwiftLM] Auto-detected VLM config (\(architecture.modelType ?? "unknown")); enabling vision mode."

tests/SwiftLMTests/NoVisionFlagTests.swift

@@ -0,0 +1,16 @@
+import XCTest
+@testable import SwiftLM
+
+/// `--no-vision` loads a vision-capable checkpoint as a text-only LLM, so text-only
+/// workloads get the prompt cache (skipped for every VLM-loaded model).
+final class NoVisionFlagTests: XCTestCase {
+
+    func testNoVisionParses() throws {
+        XCTAssertTrue(try MLXServer.parse(["--model", "m", "--no-vision"]).noVision)
+        XCTAssertFalse(try MLXServer.parse(["--model", "m"]).noVision)
+    }
+
+    func testVisionAndNoVisionAreMutuallyExclusive() {
+        XCTAssertThrowsError(try MLXServer.parse(["--model", "m", "--vision", "--no-vision"]))
+    }
+}