diff --git a/app/modules/features/SettingsFeature/Sources/AISettings/AIModels/ModelsView.swift b/app/modules/features/SettingsFeature/Sources/AISettings/AIModels/ModelsView.swift index 87fd64969..9eaad4183 100644 --- a/app/modules/features/SettingsFeature/Sources/AISettings/AIModels/ModelsView.swift +++ b/app/modules/features/SettingsFeature/Sources/AISettings/AIModels/ModelsView.swift @@ -191,21 +191,20 @@ struct ModelCard: View { .padding(.top, 8) } - if let pricing = model.defaultPricing { - HStack { + HStack { + if let pricing = model.defaultPricing { Text("Pricing:") .font(.headline) .fontWeight(.medium) Text("\(displayPrice(pricing.input)) / \(displayPrice(pricing.output))") .fontWeight(.medium) - - Spacer() - - Toggle("", isOn: $isActive) - .toggleStyle(.switch) } - .padding(.top, 8) + Spacer() + + Toggle("", isOn: $isActive) + .toggleStyle(.switch) } + .padding(.top, 8) if provider.externalAgent != nil { Text("\(model.name) is an external agent") diff --git a/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/AIProviderView.swift b/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/AIProviderView.swift index c5943fba4..9cc6020a2 100644 --- a/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/AIProviderView.swift +++ b/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/AIProviderView.swift @@ -14,7 +14,7 @@ struct AIProviderView: View { viewModel: LLMSettingsViewModel, provider: AIProvider, providerSettings: AIProviderSettings?, - isConnected: Bool, + isConfigured: Bool, enabledModels: [AIModelID], onSettingsChanged: ((AIProviderSettings?) -> Void)?, onSelectModels: (() -> Void)?, @@ -23,7 +23,7 @@ struct AIProviderView: View { self.viewModel = viewModel self.provider = provider self.providerSettings = providerSettings - self.isConnected = isConnected + self.isConfigured = isConfigured self.enabledModels = enabledModels self.onSettingsChanged = onSettingsChanged self.onSelectModels = onSelectModels @@ -41,9 +41,9 @@ struct AIProviderView: View { .font(.title2) .fontWeight(.medium) Spacer() - Text(isConnected ? "Connected" : "Not connected") + Text(isConfigured ? "Configured" : "Not configured") .font(.subheadline) - .foregroundColor(isConnected ? colorScheme.addedLineDiffText : .secondary) + .foregroundColor(isConfigured ? colorScheme.addedLineDiffText : .secondary) } if let websiteURL = provider.websiteURL { @@ -102,14 +102,19 @@ struct AIProviderView: View { } } - // Local executable section (for providers that are local) + // External agent section if let externalAgent = provider.externalAgent { ExternalAgentView(externalAgent: externalAgent, executable: $executable) } + + // Local inference section + if let localInference = provider.localInference { + LocalInferenceView(localInference: localInference, baseURL: $baseURL, executable: $executable) + } } // Models button - if isConnected, let onSelectModels, provider.externalAgent == nil { + if isConfigured, let onSelectModels, !provider.isExternalAgent { Button(action: { onSelectModels() }) { @@ -140,6 +145,9 @@ struct AIProviderView: View { .onChange(of: apiKey) { _, _ in saveSettings() } + .onChange(of: baseURL) { _, _ in + saveSettings() + } .onChange(of: executable) { _, _ in saveSettings() } @@ -159,7 +167,7 @@ struct AIProviderView: View { private let provider: AIProvider private let providerSettings: AIProviderSettings? - private let isConnected: Bool + private let isConfigured: Bool private let onSettingsChanged: ((AIProviderSettings?) -> Void)? private let onSelectModels: (() -> Void)? private let frameless: Bool @@ -186,20 +194,25 @@ struct AIProviderView: View { let trimmedBaseURL = baseURL.trimmingCharacters(in: .whitespacesAndNewlines) let trimmedExecutable = executable.trimmingCharacters(in: .whitespacesAndNewlines) - if provider.externalAgent == nil { + if provider.needsAPIKey { guard !trimmedAPIKey.isEmpty else { onSettingsChanged(nil) return } - } else { + } else if provider.isExternalAgent { guard !trimmedExecutable.isEmpty else { onSettingsChanged(nil) return } + } else if provider.isLocalInference { + guard !trimmedExecutable.isEmpty || URL(string: trimmedBaseURL) != nil else { + onSettingsChanged(nil) + return + } } let providerSettings = AIProviderSettings( - apiKey: trimmedAPIKey, + apiKey: trimmedAPIKey.isEmpty ? nil : trimmedAPIKey, baseUrl: trimmedBaseURL.isEmpty ? nil : trimmedBaseURL, executable: trimmedExecutable.isEmpty ? nil : trimmedExecutable, createdOrder: -1) @@ -236,6 +249,8 @@ extension AIProvider { "Mistral" case .inception: "Inception" + case .ollama: + "Chat & build with open models" default: "Unknown provider" } @@ -243,14 +258,23 @@ extension AIProvider { /// Whether the provider requires an API key to function (regardless of whether one has already been provided). var needsAPIKey: Bool { - externalAgent == nil + !isExternalAgent && !isLocalInference } - func isConnected(_ providerSettings: AIProviderSettings?) -> Bool { - if externalAgent != nil { - providerSettings?.executable?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false - } else { - providerSettings?.apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false + func isConfigured(_ providerSettings: AIProviderSettings?) -> Bool { + if needsAPIKey { + return providerSettings?.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false + } else if isExternalAgent { + return providerSettings?.executable?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false + } else if isLocalInference { + if + let baseUrl = providerSettings?.baseUrl?.trimmingCharacters(in: .whitespacesAndNewlines), + URL(string: baseUrl) != nil + { + return true + } + return providerSettings?.executable?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false } + return false } } diff --git a/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/AIProvidersView.swift b/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/AIProvidersView.swift index 78a0b75b9..5b583a0d2 100644 --- a/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/AIProvidersView.swift +++ b/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/AIProvidersView.swift @@ -45,7 +45,7 @@ public struct AIProvidersView: View { viewModel: viewModel, provider: providerInfo.provider, providerSettings: providerInfo.settings, - isConnected: providerInfo.isConnected, + isConfigured: providerInfo.isConfigured, enabledModels: viewModel.enabledModels, onSettingsChanged: { newSettings in updateProviderSettings(for: providerInfo.provider, with: newSettings) @@ -88,7 +88,7 @@ public struct AIProvidersView: View { return ProviderInfo( provider: provider, settings: existingSettings, - isConnected: provider.isConnected(existingSettings)) + isConfigured: provider.isConfigured(existingSettings)) } return searchText.isEmpty @@ -104,7 +104,7 @@ public struct AIProvidersView: View { private func setInitialOrder() { orderedProviders = AIProvider.allCases.map { provider in - (provider, provider.isConnected(viewModel.providerSettings[provider])) + (provider, provider.isConfigured(viewModel.providerSettings[provider])) }.sorted { lhs, rhs in // Sort: connected first, then alphabetically if lhs.1 != rhs.1 { @@ -137,7 +137,7 @@ public struct AIProvidersView: View { private struct ProviderInfo { let provider: AIProvider let settings: AIProviderSettings? - let isConnected: Bool + let isConfigured: Bool } // MARK: - ProviderModelSelectionView @@ -164,7 +164,7 @@ private struct ProviderModelSelectionView: View { viewModel: viewModel, provider: provider, providerSettings: providerSettings, - isConnected: true, + isConfigured: true, enabledModels: viewModel.enabledModels, onSettingsChanged: nil, onSelectModels: nil) diff --git a/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/ExternalAgentView.swift b/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/ExternalAgentView.swift index f2b6fbda4..8ba0de1f9 100644 --- a/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/ExternalAgentView.swift +++ b/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/ExternalAgentView.swift @@ -20,7 +20,7 @@ struct ExternalAgentView: View { provider = externalAgent.llmProvider self.externalAgent = externalAgent _executable = executable - _executableFinder = .init(initialValue: ExecutableFinder(defaultExecutable: externalAgent.defaultExecutableName)) + _executableFinder = .init(initialValue: ExecutableFinder(executable: externalAgent.defaultExecutableName)) } /// The external agent configuration. @@ -119,32 +119,6 @@ struct ExternalAgentView: View { @Environment(\.colorScheme) private var colorScheme } -// MARK: - ExecutableFinder - -/// A helper that finds where a given executable is located on disk by running `which`. -@MainActor @Observable -private final class ExecutableFinder { - /// Initializes the finder and attempts to locate the executable using `which`. - init(defaultExecutable: String) { - @Dependency(\.shellService) var shellService - - Task { [weak self] in - do { - let executablePath = try await shellService.runAndThrow("which \(defaultExecutable)", useInteractiveShell: true) - await MainActor.run { - guard let self else { return } - self.executablePath = executablePath?.trimmingCharacters(in: .whitespacesAndNewlines) - } - } catch { - // Silently ignore errors - executable not found is expected - } - } - } - - /// The path where the executable was found, or nil if not found. - private(set) var executablePath: String? -} - extension ExternalAgent { /// Whether the external agent has been enabled at least once. /// When the agent is disabled, this value will help understand whether the agent has never been enabled, and can be enabled by default, or if it has been disabled by the user. diff --git a/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/LocalInferenceView.swift b/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/LocalInferenceView.swift new file mode 100644 index 000000000..4a405c560 --- /dev/null +++ b/app/modules/features/SettingsFeature/Sources/AISettings/AIProviders/LocalInferenceView.swift @@ -0,0 +1,95 @@ +// Copyright cmd app, Inc. Licensed under the Apache License, Version 2.0. +// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +import Dependencies +import DLS +import LLMFoundation +import SwiftUI + +// MARK: - LocalInferenceView + +struct LocalInferenceView: View { + /// Initializes the card with the local inference configuration and a binding to its executable path. + init( + localInference: LocalInference, + baseURL: Binding, + executable: Binding) + { + provider = localInference.llmProvider + self.localInference = localInference + _baseURL = baseURL + _executable = executable + _executableFinder = .init(initialValue: ExecutableFinder(executable: localInference.executableName)) + } + + /// The local inference configuration. + let localInference: LocalInference + + /// The AI provider associated with this local inference. + let provider: AIProvider + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + if let executablePath = executableFinder.executablePath { + HStack { + Text("\(provider.name)'s executable was found") + .fontWeight(.medium) + + Spacer() + + HoveredButton( + action: { + executable = executable.isEmpty ? executablePath : "" + }, + onHoverColor: colorScheme.tertiarySystemBackground, + backgroundColor: colorScheme.secondarySystemBackground, + padding: 5, + content: { + Text(executable.isEmpty ? "Enable" : "Disable") + }) + } + } else if baseURL.isEmpty { + Text("\(provider.name)'s executable could not be found. Either:") + .font(.subheadline) + .fontWeight(.medium) + PlainLink("install it first", destination: localInference.installationInstructions) + .font(.subheadline) + .fontWeight(.medium) + Text("or configure a custom base URL below:") + .font(.subheadline) + .fontWeight(.medium) + } + + let showTitle = executableFinder.executablePath != nil || !baseURL.isEmpty + if showTitle { + Text("Base URL") + .font(.subheadline) + .fontWeight(.medium) + } + TextField(localInference.defaultBaseUrl.absoluteString, text: $baseURL) + .textFieldStyle(.plain) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .with( + cornerRadius: 6, + backgroundColor: Color( + NSColor.textBackgroundColor), + borderColor: Color.gray.opacity(0.3)) + + Text("Leave empty to use default") + .font(.footnote) + .foregroundColor(.secondary) + } + } + + /// Binding to the executable path or launch command. + @Binding private var executable: String + + /// Binding to the base URL. + @Binding private var baseURL: String + + /// Helper to locate the executable on disk. + @State private var executableFinder: ExecutableFinder + + @Environment(\.colorScheme) private var colorScheme +} diff --git a/app/modules/features/SettingsFeature/Sources/CodeCompletionSettingsView.swift b/app/modules/features/SettingsFeature/Sources/CodeCompletionSettingsView.swift index c1a3dc8b4..01a841185 100644 --- a/app/modules/features/SettingsFeature/Sources/CodeCompletionSettingsView.swift +++ b/app/modules/features/SettingsFeature/Sources/CodeCompletionSettingsView.swift @@ -225,7 +225,7 @@ private struct CodeCompletionProviderSection: View { viewModel: llmSettingsViewModel, provider: aiProvider, providerSettings: llmSettingsViewModel.providerSettings[aiProvider], - isConnected: aiProvider.isConnected(llmSettingsViewModel.providerSettings[aiProvider]), + isConfigured: aiProvider.isConfigured(llmSettingsViewModel.providerSettings[aiProvider]), enabledModels: llmSettingsViewModel.enabledModels, onSettingsChanged: { newSettings in updateProviderSettings(for: aiProvider, with: newSettings) diff --git a/app/modules/features/SettingsFeature/Sources/ExecutableFinder.swift b/app/modules/features/SettingsFeature/Sources/ExecutableFinder.swift new file mode 100644 index 000000000..2bcd61b4e --- /dev/null +++ b/app/modules/features/SettingsFeature/Sources/ExecutableFinder.swift @@ -0,0 +1,32 @@ +// Copyright cmd app, Inc. Licensed under the Apache License, Version 2.0. +// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +import Dependencies +import ShellServiceInterface +import SwiftUI + +// MARK: - ExecutableFinder + +/// A helper that finds where a given executable is located on disk by running `which`. +@MainActor @Observable +final class ExecutableFinder { + /// Initializes the finder and attempts to locate the executable using `which`. + init(executable: String) { + @Dependency(\.shellService) var shellService + + Task { [weak self] in + do { + let executablePath = try await shellService.runAndThrow("which \(executable)", useInteractiveShell: true) + await MainActor.run { + guard let self else { return } + self.executablePath = executablePath?.trimmingCharacters(in: .whitespacesAndNewlines) + } + } catch { + // Silently ignore errors - executable not found is expected + } + } + } + + /// The path where the executable was found, or nil if not found. + private(set) var executablePath: String? +} diff --git a/app/modules/foundations/LLMFoundation/Sources/AIModel.swift b/app/modules/foundations/LLMFoundation/Sources/AIModel.swift index 7c6c4d977..08c595dc7 100644 --- a/app/modules/foundations/LLMFoundation/Sources/AIModel.swift +++ b/app/modules/foundations/LLMFoundation/Sources/AIModel.swift @@ -5,7 +5,9 @@ import Foundation // MARK: - LLMReasoning -public struct LLMReasoning: Sendable, Hashable, Codable { } +public struct LLMReasoning: Sendable, Hashable, Codable { + public init() { } +} // MARK: - AIProviderModel @@ -38,11 +40,14 @@ public struct AIModel: Hashable, Identifiable, Sendable, Codable { description: String? = nil, contextSize: Int, maxOutputTokens: Int, - defaultPricing: ModelPricing?, // TODO: Make non-optional when we have pricing for all models. + defaultPricing: ModelPricing?, documentationURL: URL? = nil, reasoning: LLMReasoning? = nil, createdAt: TimeInterval, - rankForProgramming: Int) + rankForProgramming: Int, + supportsChat: Bool = true, + supportsTools: Bool = true, + supportsCompletion: Bool = false) { self.slug = slug self.name = name @@ -54,6 +59,13 @@ public struct AIModel: Hashable, Identifiable, Sendable, Codable { self.reasoning = reasoning self.createdAt = createdAt self.rankForProgramming = rankForProgramming + self.supportsChat = supportsChat + // TODO: move to appropriate place + && AIModel.modelSupportsChat(id: slug) + self.supportsTools = supportsTools + self.supportsCompletion = supportsCompletion + // TODO: move to appropriate place + || AIModel.modelSupportsCompletion(id: slug) } /// A few models for debugging and providing default values. @@ -122,6 +134,9 @@ public struct AIModel: Hashable, Identifiable, Sendable, Codable { public let reasoning: LLMReasoning? public let createdAt: TimeInterval public let rankForProgramming: Int + public let supportsChat: Bool + public let supportsTools: Bool + public let supportsCompletion: Bool public var id: String { slug @@ -147,11 +162,11 @@ public struct AIModel: Hashable, Identifiable, Sendable, Codable { } extension AIModel { - public var supportsCompletion: Bool { + static func modelSupportsCompletion(id: String) -> Bool { ["inception/mercury-coder-small", "mistralai/codestral-latest"].contains(id) } - public var supportsChat: Bool { + static func modelSupportsChat(id: String) -> Bool { id != "inception/mercury-coder-small" } } diff --git a/app/modules/foundations/LLMFoundation/Sources/AIProvider.swift b/app/modules/foundations/LLMFoundation/Sources/AIProvider.swift index 31ce879f6..235ef727a 100644 --- a/app/modules/foundations/LLMFoundation/Sources/AIProvider.swift +++ b/app/modules/foundations/LLMFoundation/Sources/AIProvider.swift @@ -17,11 +17,11 @@ public struct AIProvider: Hashable, Identifiable, CaseIterable, Sendable, RawRep init( id: String, name: String, - keychainKey: String, + keychainKey: String? = nil, websiteURL: URL? = nil, apiKeyCreationURL: URL? = nil, lowTierModelId: AIModelID? = nil, - modelsEnabledByDefault: [AIModelID]) + modelsEnabledByDefault: [AIModelID] = []) { self.id = id self.name = name @@ -44,12 +44,13 @@ public struct AIProvider: Hashable, Identifiable, CaseIterable, Sendable, RawRep .geminiCLI, .mistral, .inception, + .ollama, ] } public let id: String public let name: String - public let keychainKey: String + public let keychainKey: String? public let websiteURL: URL? public let apiKeyCreationURL: URL? public let lowTierModelId: AIModelID? diff --git a/app/modules/foundations/LLMFoundation/Sources/LocalInference.swift b/app/modules/foundations/LLMFoundation/Sources/LocalInference.swift new file mode 100644 index 000000000..3a14ab400 --- /dev/null +++ b/app/modules/foundations/LLMFoundation/Sources/LocalInference.swift @@ -0,0 +1,41 @@ +// Copyright cmd app, Inc. Licensed under the Apache License, Version 2.0. +// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +import Foundation + +// MARK: - LocalInference + +/// Inference with local LLMs (e.g Ollama). +public struct LocalInference: Sendable { + /// The name of the inference. + public let name: String + /// An executable name + public let executableName: String + /// A base URL for a local AP.. + public let defaultBaseUrl: URL + /// A link to instructions on how to install. + public let installationInstructions: URL + /// Additional information about the associated provider. + public let llmProvider: AIProvider +} + +extension AIProvider { + public var localInference: LocalInference? { + switch self { + case .ollama: + .init( + name: "Ollama", + executableName: "ollama", + defaultBaseUrl: URL(string: "http://localhost:11434")!, + installationInstructions: URL(string: "https://docs.ollama.com/quickstart")!, + llmProvider: self) + + default: + nil + } + } + + public var isLocalInference: Bool { + localInference != nil + } +} diff --git a/app/modules/foundations/LLMFoundation/Sources/providers/Ollama.swift b/app/modules/foundations/LLMFoundation/Sources/providers/Ollama.swift new file mode 100644 index 000000000..1736c3322 --- /dev/null +++ b/app/modules/foundations/LLMFoundation/Sources/providers/Ollama.swift @@ -0,0 +1,12 @@ +// Copyright cmd app, Inc. Licensed under the Apache License, Version 2.0. +// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +import AppFoundation +import Foundation + +extension AIProvider { + public static let ollama = AIProvider( + id: "ollama", + name: "Ollama", + websiteURL: URL(string: "https://ollama.com/search")) +} diff --git a/app/modules/serviceInterfaces/LocalServerServiceInterface/Sources/listModelsSchema.generated.swift b/app/modules/serviceInterfaces/LocalServerServiceInterface/Sources/listModelsSchema.generated.swift index 2af431995..692560dcf 100644 --- a/app/modules/serviceInterfaces/LocalServerServiceInterface/Sources/listModelsSchema.generated.swift +++ b/app/modules/serviceInterfaces/LocalServerServiceInterface/Sources/listModelsSchema.generated.swift @@ -61,9 +61,13 @@ extension Schema { public let maxCompletionTokens: Int public let inputModalities: [ModelModality] public let outputModalities: [ModelModality] - public let pricing: ModelPricing + public let pricing: ModelPricing? public let createdAt: Double public let rankForProgramming: Int + public let supportsChat: Bool + public let supportsTools: Bool + public let supportsReasoning: Bool + public let supportsCompletion: Bool private enum CodingKeys: String, CodingKey { case providerId = "providerId" @@ -77,6 +81,10 @@ extension Schema { case pricing = "pricing" case createdAt = "createdAt" case rankForProgramming = "rankForProgramming" + case supportsChat = "supportsChat" + case supportsTools = "supportsTools" + case supportsReasoning = "supportsReasoning" + case supportsCompletion = "supportsCompletion" } public init( @@ -88,9 +96,13 @@ extension Schema { maxCompletionTokens: Int, inputModalities: [ModelModality], outputModalities: [ModelModality], - pricing: ModelPricing, + pricing: ModelPricing? = nil, createdAt: Double, - rankForProgramming: Int + rankForProgramming: Int, + supportsChat: Bool, + supportsTools: Bool, + supportsReasoning: Bool, + supportsCompletion: Bool ) { self.providerId = providerId self.globalId = globalId @@ -103,6 +115,10 @@ extension Schema { self.pricing = pricing self.createdAt = createdAt self.rankForProgramming = rankForProgramming + self.supportsChat = supportsChat + self.supportsTools = supportsTools + self.supportsReasoning = supportsReasoning + self.supportsCompletion = supportsCompletion } public init(from decoder: Decoder) throws { @@ -115,9 +131,13 @@ extension Schema { maxCompletionTokens = try container.decode(Int.self, forKey: .maxCompletionTokens) inputModalities = try container.decode([ModelModality].self, forKey: .inputModalities) outputModalities = try container.decode([ModelModality].self, forKey: .outputModalities) - pricing = try container.decode(ModelPricing.self, forKey: .pricing) + pricing = try container.decodeIfPresent(ModelPricing?.self, forKey: .pricing) createdAt = try container.decode(Double.self, forKey: .createdAt) rankForProgramming = try container.decode(Int.self, forKey: .rankForProgramming) + supportsChat = try container.decode(Bool.self, forKey: .supportsChat) + supportsTools = try container.decode(Bool.self, forKey: .supportsTools) + supportsReasoning = try container.decode(Bool.self, forKey: .supportsReasoning) + supportsCompletion = try container.decode(Bool.self, forKey: .supportsCompletion) } public func encode(to encoder: Encoder) throws { @@ -130,9 +150,13 @@ extension Schema { try container.encode(maxCompletionTokens, forKey: .maxCompletionTokens) try container.encode(inputModalities, forKey: .inputModalities) try container.encode(outputModalities, forKey: .outputModalities) - try container.encode(pricing, forKey: .pricing) + try container.encodeIfPresent(pricing, forKey: .pricing) try container.encode(createdAt, forKey: .createdAt) try container.encode(rankForProgramming, forKey: .rankForProgramming) + try container.encode(supportsChat, forKey: .supportsChat) + try container.encode(supportsTools, forKey: .supportsTools) + try container.encode(supportsReasoning, forKey: .supportsReasoning) + try container.encode(supportsCompletion, forKey: .supportsCompletion) } } public enum ModelModality: String, Codable, Sendable, CaseIterable { diff --git a/app/modules/serviceInterfaces/LocalServerServiceInterface/Sources/sendMessageSchema.generated.swift b/app/modules/serviceInterfaces/LocalServerServiceInterface/Sources/sendMessageSchema.generated.swift index 8fbc00c1c..b6449a06c 100644 --- a/app/modules/serviceInterfaces/LocalServerServiceInterface/Sources/sendMessageSchema.generated.swift +++ b/app/modules/serviceInterfaces/LocalServerServiceInterface/Sources/sendMessageSchema.generated.swift @@ -10,7 +10,7 @@ extension Schema { public let messages: [Message] public let system: String? public let projectRoot: String? - public let tools: [Tool] + public let tools: [Tool]? public let model: String public let enableReasoning: Bool public let provider: APIProvider @@ -31,7 +31,7 @@ extension Schema { messages: [Message], system: String? = nil, projectRoot: String? = nil, - tools: [Tool], + tools: [Tool]? = nil, model: String, enableReasoning: Bool, provider: APIProvider, @@ -52,7 +52,7 @@ extension Schema { messages = try container.decode([Message].self, forKey: .messages) system = try container.decodeIfPresent(String?.self, forKey: .system) projectRoot = try container.decodeIfPresent(String?.self, forKey: .projectRoot) - tools = try container.decode([Tool].self, forKey: .tools) + tools = try container.decodeIfPresent([Tool]?.self, forKey: .tools) model = try container.decode(String.self, forKey: .model) enableReasoning = try container.decode(Bool.self, forKey: .enableReasoning) provider = try container.decode(APIProvider.self, forKey: .provider) @@ -64,7 +64,7 @@ extension Schema { try container.encode(messages, forKey: .messages) try container.encodeIfPresent(system, forKey: .system) try container.encodeIfPresent(projectRoot, forKey: .projectRoot) - try container.encode(tools, forKey: .tools) + try container.encodeIfPresent(tools, forKey: .tools) try container.encode(model, forKey: .model) try container.encode(enableReasoning, forKey: .enableReasoning) try container.encode(provider, forKey: .provider) @@ -810,6 +810,7 @@ extension Schema { case gemini = "gemini" case mistral = "mistral" case inception = "inception" + case ollama = "ollama" case claudeCode = "claude_code" case codex = "codex" case geminiCli = "gemini_cli" diff --git a/app/modules/serviceInterfaces/SettingsServiceInterface/Sources/SettingsService.swift b/app/modules/serviceInterfaces/SettingsServiceInterface/Sources/SettingsService.swift index 69d1c6695..e6ebd9ab4 100644 --- a/app/modules/serviceInterfaces/SettingsServiceInterface/Sources/SettingsService.swift +++ b/app/modules/serviceInterfaces/SettingsServiceInterface/Sources/SettingsService.swift @@ -154,12 +154,12 @@ public struct Settings: Sendable, Equatable { /// To help keep track of which Provider was setup first, we use an incrementing order. /// This order can be useful for determining which provider to default to when multiple are available. public let createdOrder: Int - public var apiKey: String + public var apiKey: String? public var baseUrl: String? public var executable: String? public init( - apiKey: String, + apiKey: String?, baseUrl: String?, executable: String?, createdOrder: Int) diff --git a/app/modules/services/LLMService/Sources/DefaultLLMService.swift b/app/modules/services/LLMService/Sources/DefaultLLMService.swift index a0ade6770..897fd1790 100644 --- a/app/modules/services/LLMService/Sources/DefaultLLMService.swift +++ b/app/modules/services/LLMService/Sources/DefaultLLMService.swift @@ -267,12 +267,14 @@ final class DefaultLLMService: LLMService { messages: messageHistory, system: system, projectRoot: context?.projectRoot?.path, - tools: tools + tools: model.supportsTools + ? tools // Unless we are using an external agent, only send to the AI tools that are internal. .filter { ($0.canBeExecuted && $0.id == $0.referenceId) || provider.isExternalAgent } - .map { .init(name: $0.name, description: $0.description, inputSchema: $0.inputSchema) }, + .map { .init(name: $0.name, description: $0.description, inputSchema: $0.inputSchema) } + : nil, model: providerModel.id, - enableReasoning: enableReasoning, + enableReasoning: model.canReason && enableReasoning, provider: .init( provider: provider, settings: providerSettings, @@ -420,6 +422,8 @@ extension Schema.APIProvider { return .mistral case .inception: return .inception + case .ollama: + return .ollama default: throw AppError(message: "Unsupported provider \(provider.name)") } diff --git a/app/modules/services/LLMService/Sources/LLMModelManager.swift b/app/modules/services/LLMService/Sources/LLMModelManager.swift index edc12315f..ed73f793f 100644 --- a/app/modules/services/LLMService/Sources/LLMModelManager.swift +++ b/app/modules/services/LLMService/Sources/LLMModelManager.swift @@ -329,21 +329,31 @@ final class AIModelsManager: AIModelsManagerProtocol { name: apiProvider.name, settings: apiProvider.settings))) let response: Schema.ListModelsOutput = try await localServer.postRequest(path: "models", data: data) - return response.models.map { AIProviderModel( - providerId: $0.providerId, - provider: provider, - modelInfo: .init( - name: $0.name, - slug: $0.globalId, - contextSize: $0.contextLength, - maxOutputTokens: $0.maxCompletionTokens, - defaultPricing: .init( - input: $0.pricing.prompt, - output: $0.pricing.completion, - cacheWrite: $0.pricing.inputCacheWrite ?? 0, - cachedInput: $0.pricing.inputCacheRead ?? 0), - createdAt: $0.createdAt, - rankForProgramming: $0.rankForProgramming)) } + return response.models.map { model in + let defaultPricing: ModelPricing? = { + guard let pricing = model.pricing else { return nil } + return .init( + input: pricing.prompt, + output: pricing.completion, + cacheWrite: pricing.inputCacheWrite ?? 0, + cachedInput: pricing.inputCacheRead ?? 0) + }() + return AIProviderModel( + providerId: model.providerId, + provider: provider, + modelInfo: .init( + name: model.name, + slug: model.globalId, + contextSize: model.contextLength, + maxOutputTokens: model.maxCompletionTokens, + defaultPricing: defaultPricing, + reasoning: model.supportsReasoning ? LLMReasoning() : nil, + createdAt: model.createdAt, + rankForProgramming: model.rankForProgramming, + supportsChat: model.supportsChat, + supportsTools: model.supportsTools, + supportsCompletion: model.supportsCompletion)) + } } private func observerChangesToSettings() { diff --git a/app/modules/services/LLMService/Tests/LLMModelManagerTests.swift b/app/modules/services/LLMService/Tests/LLMModelManagerTests.swift index 12e43bfb7..b2cd3a46a 100644 --- a/app/modules/services/LLMService/Tests/LLMModelManagerTests.swift +++ b/app/modules/services/LLMService/Tests/LLMModelManagerTests.swift @@ -1098,7 +1098,11 @@ private func makeSchemaModel( prompt: 1.0, completion: 2.0), createdAt: Date().timeIntervalSince1970, - rankForProgramming: 1) + rankForProgramming: 1, + supportsChat: true, + supportsTools: true, + supportsReasoning: false, + supportsCompletion: true) } private func makeListModelsOutput(models: [Schema.Model]) -> Schema.ListModelsOutput { diff --git a/app/modules/services/SettingsService/Sources/DefaultSettingsService.swift b/app/modules/services/SettingsService/Sources/DefaultSettingsService.swift index f3e469cbf..f1a22c5c1 100644 --- a/app/modules/services/SettingsService/Sources/DefaultSettingsService.swift +++ b/app/modules/services/SettingsService/Sources/DefaultSettingsService.swift @@ -270,7 +270,7 @@ final class DefaultSettingsService: SettingsService { let keychainKeyPrefix = "cmd-keychain-key-" for provider in AIProvider.allCases { - let keychainKey = keychainKeyPrefix + provider.keychainKey + guard let keychainKey = provider.keychainKey.map({ keychainKeyPrefix + $0 }) else { continue } if let settings = settings.llmProviderSettings[provider] { privateKeys[keychainKey] = settings.apiKey publicSettings.llmProviderSettings[provider]?.apiKey = keychainKey diff --git a/contributing.md b/contributing.md index 928ead7cd..d5d63a248 100644 --- a/contributing.md +++ b/contributing.md @@ -55,6 +55,7 @@ export OPENAI_LOCAL_SERVER_PROXY="http://localhost:10003/v1" export GROQ_LOCAL_SERVER_PROXY="http://localhost:10004/openai/v1" export GEMINI_LOCAL_SERVER_PROXY="http://localhost:10005/v1beta" export GITHUB_COPILOT_PROXY="http://localhost:9090" +export OLLAMA_LOCAL_SERVER_PROXY="http://localhost:10006" # Claude Code (with Proxyman): cat > "$HOME/.claude/start_with_proxy.sh" << 'EOF' diff --git a/docs/docs.json b/docs/docs.json index 73d2c4c4c..754bbe413 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -26,6 +26,7 @@ "group": "Configuration", "pages": [ "pages/ai-providers", + "pages/ollama", "pages/models", "pages/code-completion", "pages/chat-modes", diff --git a/docs/pages/ai-providers.mdx b/docs/pages/ai-providers.mdx index 7faa6dfbe..97a746255 100644 --- a/docs/pages/ai-providers.mdx +++ b/docs/pages/ai-providers.mdx @@ -5,12 +5,13 @@ description: "Configure AI providers to use with cmd" ## Overview -cmd connects to AI providers using your API keys. You can configure multiple providers and switch between them as needed. This gives you flexibility to: +cmd connects to AI providers using your API keys, or use local inference with providers like Ollama. You can configure multiple providers and switch between them as needed. This gives you flexibility to: - Use different providers for different tasks +- Run AI models locally for privacy and offline development - Have backup options if one provider has issues - Compare performance across providers -- Take advantage of different pricing models +- Take advantage of different pricing models or eliminate API costs entirely ## Initial Setup @@ -61,6 +62,12 @@ When multiple providers offer the same model, you can set which provider to use +### Local Inference + +For complete data privacy and offline development, you can use [Ollama](/pages/ollama) to run AI models locally on your machine or on a private network server. This eliminates API costs and ensures your code never leaves your infrastructure. + +See the [Ollama configuration guide](/pages/ollama) for setup instructions. + Configure multiple providers to compare costs and performance. You can switch providers anytime from **Settings > Models**. diff --git a/docs/pages/models.mdx b/docs/pages/models.mdx index 3652d40f0..19cc2d7cb 100644 --- a/docs/pages/models.mdx +++ b/docs/pages/models.mdx @@ -10,6 +10,7 @@ After configuring your [AI providers](/pages/ai-providers), you need to enable w - Which models are available in the model selector - Which provider to use when a model is available from multiple sources - Default model preferences +- Local models from [Ollama](/pages/ollama) for private, offline inference You can configure models during onboarding or anytime through **Settings > Models** or **Settings > AI Providers**. @@ -43,6 +44,7 @@ Some models may be available from multiple providers. For example: - **Claude Sonnet** from both Anthropic and OpenRouter - **GPT-4** from both OpenAI and Azure OpenAI - Popular models from both direct providers and aggregators +- Local models from Ollama vs. cloud providers ### Setting Provider Preference @@ -87,3 +89,28 @@ Once you've enabled models, you can switch between them easily in the chat inter The model selector shows only the models you've enabled in settings. After changing model mid-conversation, the next message will not have a cached prompt + +## Local Models + +For complete privacy and offline development, you can use local models through [Ollama](/pages/ollama). Once you've configured the Ollama provider, all models installed in your Ollama instance will automatically appear in the model list. + + + + Use the Ollama CLI to install models: + ```bash + ollama pull qwen2.5-coder:7b + ``` + + + + If you've added new models to Ollama, refresh cmd's model list by disabling and re-enabling the Ollama provider in **Settings > AI Providers** + + + + Go to **Settings > Models** and enable the Ollama models you want to use + + + + + Local models run entirely on your machine, ensuring your code never leaves your infrastructure. This is ideal for sensitive projects or working offline. + diff --git a/docs/pages/ollama.mdx b/docs/pages/ollama.mdx new file mode 100644 index 000000000..7cb088c64 --- /dev/null +++ b/docs/pages/ollama.mdx @@ -0,0 +1,166 @@ +--- +title: "Ollama" +description: "Configure Ollama for local AI inference with cmd" +--- + +## Overview + +Ollama enables you to run AI models locally on your machine or on a private network endpoint. This gives you complete control over your data, eliminates API costs, and enables offline development. Ollama can automatically download and manage models for you, making local inference accessible and straightforward. + +## Prerequisites + +Before configuring Ollama in cmd, you need to have Ollama installed and running on your system. + + + + Download and install Ollama from the [official website](https://ollama.com/) or follow the [Quick Start guide](https://docs.ollama.com/quickstart). + + + + Ensure Ollama is running. The default installation sets up autostart, but you can also start it manually: + + ```bash + ollama serve + ``` + + Ollama will run on `http://localhost:11434` by default. + + + + Install one or more AI models. Ollama will automatically download the models when you first request them: + + ```bash + # Example: Install a coding-focused model + ollama pull devstral-small-2:24b + + # Or install a small model with lower system requirements + ollama pull qwen2.5-coder:7b + + # Or install a reasoning model + ollama pull deepseek-r1:14b + ``` + + You can browse available models at [https://ollama.com/search](https://ollama.com/search). + + + + Verify Ollama is working correctly: + + ```bash + ollama list + ``` + + This should show your installed models. + + + + + Ollama automatically loads models on-demand. You don't need to manually load models before using them with cmd. + + +## Configuring Ollama in cmd + +Once Ollama is installed and running, you can enable it in cmd: + + + + Go to **Settings > AI Providers** in cmd + + + + Click **"Add Provider"** or **"Configure"** for Ollama. + + By default, cmd connects to Ollama on `http://localhost:11434`. If you're running Ollama on a different port or on a remote server, you can customize the base URL. + + + + For a standard local Ollama installation, simply toggle **"Enable"** + + + + If you're running Ollama on a different port: + + ``` + Base URL: http://localhost:8080 + ``` + + + + If you're running Ollama on a remote server in your private network: + + ``` + Base URL: http://192.168.1.100:11434 + ``` + + + + + + cmd will automatically discover the available models from your Ollama installation when the provider is configured. + + + + Go to **Settings > Models** and enable the models you want to use in cmd. All models installed in Ollama will be available for selection. + + + + + If you install new models in Ollama after configuring the provider, you can refresh the model list by disabling and re-enabling the Ollama provider in **Settings > AI Providers**. + + +## Using Ollama + +Once enabled and models are selected, you can use Ollama models just like any other AI provider: + + + + In the cmd interface, open the model selector and choose one of your Ollama models (e.g., "devstral-small-2:24b") + + + + You can now use Ollama models for: + - Chat and code assistance + - Agent mode for autonomous tasks (if supported by the model) + - All other cmd features such as code completion + + All inference happens locally on your machine or your private server - no data leaves your network. + + + +## Tested Models + +The following models have been successfully tested with cmd: + +- **qwen3-coder:30b** - High-performance coding model +- **devstral-small-2:24b** - Efficient development assistant +- **deepseek-r1:14b** - Reasoning-focused model +- **qwen2.5-coder:7b** - Lightweight coding model + +You can find more models at [https://ollama.com/search](https://ollama.com/search). + +## Benefits of Local Inference + +Using Ollama with cmd provides several advantages: + + + + All processing happens locally - your code never leaves your machine + + + Run unlimited inference without per-token charges + + + Work without internet connectivity + + + Use specialized or fine-tuned models for your specific needs + + + + + Local models require significant computational resources. Larger models provide better results but need more RAM and processing power. Ensure your system meets the requirements for the models you want to run. + + + + For the best coding experience, consider models specifically trained for code. These models understand programming contexts better than general-purpose models. + diff --git a/docs/snippets/features-showcase.mdx b/docs/snippets/features-showcase.mdx index f2a280251..d077467c7 100644 --- a/docs/snippets/features-showcase.mdx +++ b/docs/snippets/features-showcase.mdx @@ -77,6 +77,19 @@ Learn how to configure External Agents → + + Run AI models entirely on your machine with Ollama: + - **Complete privacy**: Your code never leaves your infrastructure + - **Custom models**: Use specialized or fine-tuned models + - **Network deployment**: Run Ollama on private network servers + - **Offline development**: Work without internet connectivity + - **No API costs**: Unlimited inference without per-token charges + + Tested successfully with models like devstral-small-2 and qwen3-coder. + + Learn how to configure Ollama → + +