refactor: repartition server-side and client-side code

app/.server/modules/llm/base-provider.ts

@@ -0,0 +1,83 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import type { LanguageModel } from 'ai';
+import type { IProviderSetting } from '~/types/model';
+import type { ModelInfo, ProviderInfo } from './types';
+
+export abstract class BaseProvider implements ProviderInfo {
+  abstract name: string;
+  abstract staticModels: ModelInfo[];
+  cachedDynamicModels?: {
+    cacheId: string;
+    models: ModelInfo[];
+  };
+
+  getApiKeyLink?: string;
+  labelForGetApiKey?: string;
+  icon?: string;
+
+  getProviderBaseUrlAndKey(providerSettings?: IProviderSetting) {
+    let baseUrl = providerSettings?.baseUrl;
+    if (baseUrl && baseUrl.endsWith('/')) {
+      baseUrl = baseUrl.slice(0, -1);
+    }
+
+    const apiKey = providerSettings?.apiKey;
+
+    return {
+      baseUrl,
+      apiKey,
+    };
+  }
+
+  getModelsFromCache(options: { providerSettings?: Record<string, IProviderSetting> }): ModelInfo[] | null {
+    if (!this.cachedDynamicModels) {
+      // console.log('no dynamic models',this.name);
+      return null;
+    }
+
+    const cacheKey = this.cachedDynamicModels.cacheId;
+    const generatedCacheKey = this.getDynamicModelsCacheKey(options);
+
+    if (cacheKey !== generatedCacheKey) {
+      // console.log('cache key mismatch',this.name,cacheKey,generatedCacheKey);
+      this.cachedDynamicModels = undefined;
+      return null;
+    }
+
+    return this.cachedDynamicModels.models;
+  }
+  getDynamicModelsCacheKey(options: { providerSettings?: Record<string, IProviderSetting> }) {
+    return JSON.stringify({
+      apiKeys: options.providerSettings?.[this.name]?.apiKey,
+      providerSettings: options.providerSettings?.[this.name],
+    });
+  }
+  storeDynamicModels(options: { providerSettings?: Record<string, IProviderSetting> }, models: ModelInfo[]) {
+    const cacheId = this.getDynamicModelsCacheKey(options);
+
+    // console.log('caching dynamic models',this.name,cacheId);
+    this.cachedDynamicModels = {
+      cacheId,
+      models,
+    };
+  }
+
+  // Declare the optional getDynamicModels method
+  getDynamicModels?(settings?: IProviderSetting): Promise<ModelInfo[]>;
+
+  abstract getModelInstance(options: {
+    model: string;
+    providerSettings?: Record<string, IProviderSetting>;
+  }): LanguageModel;
+}
+
+type OptionalApiKey = string | undefined;
+
+export function getOpenAILikeModel(baseURL: string, apiKey: OptionalApiKey, model: string) {
+  const openai = createOpenAI({
+    baseURL,
+    apiKey,
+  });
+
+  return openai(model);
+}

app/.server/modules/llm/manager.server.ts

@@ -0,0 +1,234 @@
+import type { IProviderSetting } from '~/types/model';
+import { createScopedLogger } from '~/utils/logger';
+import { BaseProvider } from './base-provider';
+import * as providers from './registry';
+import type { ModelInfo, ProviderInfo } from './types';
+
+const logger = createScopedLogger('LLMManager');
+
+export class LLMManager {
+  private static _instance: LLMManager;
+  private _providers: Map<string, BaseProvider> = new Map();
+  private _modelList: ModelInfo[] = [];
+
+  constructor() {
+    this._registerProvidersFromDirectory();
+  }
+
+  static getInstance(): LLMManager {
+    if (!LLMManager._instance) {
+      LLMManager._instance = new LLMManager();
+    }
+
+    return LLMManager._instance;
+  }
+
+  private _getEnvConfig<T>(key: string, defaultValue: T): T {
+    const value = process?.env?.[key] || (import.meta.env as any)?.[key];
+
+    if (value === undefined || value === null || value === '') {
+      return defaultValue;
+    }
+
+    if (typeof defaultValue === 'boolean') {
+      return (value === 'true' || value === true) as unknown as T;
+    }
+
+    if (typeof defaultValue === 'number') {
+      return Number(value) as unknown as T;
+    }
+
+    if (Array.isArray(defaultValue)) {
+      return (value
+        ? String(value)
+            .split(',')
+            .map((item) => item.trim())
+        : []) as unknown as T;
+    }
+
+    return value as T;
+  }
+
+  private _getUnifiedProviderConfig() {
+    const providerName = this._getEnvConfig<string>('LLM_PROVIDER', '');
+    const baseUrl = this._getEnvConfig<string>('PROVIDER_BASE_URL', '');
+    const apiKey = this._getEnvConfig<string>('PROVIDER_API_KEY', '');
+
+    return {
+      providerName,
+      baseUrl,
+      apiKey,
+    };
+  }
+
+  getDefaultProvider(): BaseProvider {
+    const { providerName } = this._getUnifiedProviderConfig();
+
+    if (!providerName || !this._providers.has(providerName)) {
+      throw new Error(
+        `Provider ${providerName} not found, Effective Provider: ${Array.from(this._providers.values())
+          .map((p) => p.name)
+          .join(', ')}`,
+      );
+    }
+
+    return this._providers.get(providerName)!;
+  }
+
+  private _registerProvidersFromDirectory() {
+    const allProviders: BaseProvider[] = Object.values(providers).map((providerClass) => new providerClass());
+    for (const provider of allProviders) {
+      this.registerProvider(provider);
+    }
+  }
+
+  registerProvider(provider: BaseProvider) {
+    if (this._providers.has(provider.name)) {
+      logger.warn(`Provider ${provider.name} is already registered. Skipping.`);
+      return;
+    }
+
+    this._providers.set(provider.name, provider);
+    this._modelList = [...this._modelList, ...provider.staticModels];
+  }
+
+  getDefaultModel(): string {
+    return this._getEnvConfig<string>('LLM_DEFAULT_MODEL', '');
+  }
+
+  getMinorModel(): string {
+    return this._getEnvConfig<string>('LLM_MINOR_MODEL', '');
+  }
+
+  getConfiguredProviderSettings(): Record<string, IProviderSetting> {
+    const providerSettings: Record<string, IProviderSetting> = {};
+
+    const { providerName, baseUrl, apiKey } = this._getUnifiedProviderConfig();
+
+    providerSettings[providerName] = {
+      enabled: true,
+      baseUrl,
+      apiKey,
+    };
+    return providerSettings;
+  }
+
+  getModelList(): ModelInfo[] {
+    return this._modelList;
+  }
+
+  async updateModelList(options: { providerSettings?: Record<string, IProviderSetting> }): Promise<ModelInfo[]> {
+    const { providerSettings } = options;
+
+    let enabledProviders = Array.from(this._providers.values()).map((p) => p.name);
+
+    if (providerSettings && Object.keys(providerSettings).length > 0) {
+      enabledProviders = enabledProviders.filter((p) => providerSettings[p]?.enabled);
+    }
+
+    // Get dynamic models from all providers that support them
+    const dynamicModels = await Promise.all(
+      Array.from(this._providers.values())
+        .filter((provider) => enabledProviders.includes(provider.name))
+        .filter(
+          (provider): provider is BaseProvider & Required<Pick<ProviderInfo, 'getDynamicModels'>> =>
+            !!provider.getDynamicModels,
+        )
+        .map(async (provider) => {
+          const cachedModels = provider.getModelsFromCache(options);
+
+          if (cachedModels) {
+            return cachedModels;
+          }
+
+          const dynamicModels = await provider
+            .getDynamicModels(providerSettings?.[provider.name])
+            .then((models) => {
+              logger.info(`Caching ${models.length} dynamic models for ${provider.name}`);
+              provider.storeDynamicModels(options, models);
+
+              return models;
+            })
+            .catch((err) => {
+              logger.error(`Error getting dynamic models ${provider.name} :`, err);
+              return [];
+            });
+
+          return dynamicModels;
+        }),
+    );
+    const staticModels = Array.from(this._providers.values()).flatMap((p) => p.staticModels || []);
+    const dynamicModelsFlat = dynamicModels.flat();
+    const dynamicModelKeys = dynamicModelsFlat.map((d) => `${d.name}-${d.provider}`);
+    const filteredStaticModesl = staticModels.filter((m) => !dynamicModelKeys.includes(`${m.name}-${m.provider}`));
+
+    // Combine static and dynamic models
+    const modelList = [...dynamicModelsFlat, ...filteredStaticModesl];
+    modelList.sort((a, b) => a.name.localeCompare(b.name));
+    this._modelList = modelList;
+
+    return modelList;
+  }
+  getStaticModelList() {
+    return [...this._providers.values()].flatMap((p) => p.staticModels || []);
+  }
+  async getModelListFromProvider(
+    providerArg: BaseProvider,
+    options: {
+      providerSettings?: Record<string, IProviderSetting>;
+    },
+  ): Promise<ModelInfo[]> {
+    const provider = this._providers.get(providerArg.name);
+
+    if (!provider) {
+      throw new Error(`Provider ${providerArg.name} not found`);
+    }
+
+    const staticModels = provider.staticModels || [];
+
+    if (!provider.getDynamicModels) {
+      return staticModels;
+    }
+
+    const { providerSettings } = options;
+
+    const cachedModels = provider.getModelsFromCache({
+      providerSettings,
+    });
+
+    if (cachedModels) {
+      logger.info(`Found ${cachedModels.length} cached models for ${provider.name}`);
+      return [...cachedModels, ...staticModels];
+    }
+
+    logger.info(`Getting dynamic models for ${provider.name}`);
+
+    const dynamicModels = await provider
+      .getDynamicModels?.(providerSettings?.[provider.name])
+      .then((models) => {
+        logger.info(`Got ${models.length} dynamic models for ${provider.name}`);
+        provider.storeDynamicModels(options, models);
+
+        return models;
+      })
+      .catch((err) => {
+        logger.error(`Error getting dynamic models ${provider.name} :`, err);
+        return [];
+      });
+    const dynamicModelsName = dynamicModels.map((d) => d.name);
+    const filteredStaticList = staticModels.filter((m) => !dynamicModelsName.includes(m.name));
+    const modelList = [...dynamicModels, ...filteredStaticList];
+    modelList.sort((a, b) => a.name.localeCompare(b.name));
+
+    return modelList;
+  }
+  getStaticModelListFromProvider(providerArg: BaseProvider) {
+    const provider = this._providers.get(providerArg.name);
+
+    if (!provider) {
+      throw new Error(`Provider ${providerArg.name} not found`);
+    }
+
+    return [...(provider.staticModels || [])];
+  }
+}

app/.server/modules/llm/providers/amazon-bedrock.ts

@@ -0,0 +1,104 @@
+import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock';
+import { type LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+interface AWSBedRockConfig {
+  region: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken?: string;
+}
+
+export default class AmazonBedrockProvider extends BaseProvider {
+  name = 'AmazonBedrock';
+  getApiKeyLink = 'https://console.aws.amazon.com/iam/home';
+
+  staticModels: ModelInfo[] = [
+    {
+      name: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
+      label: 'Claude 3.5 Sonnet v2 (Bedrock)',
+      provider: 'AmazonBedrock',
+      maxTokenAllowed: 200000,
+    },
+    {
+      name: 'anthropic.claude-3-5-sonnet-20240620-v1:0',
+      label: 'Claude 3.5 Sonnet (Bedrock)',
+      provider: 'AmazonBedrock',
+      maxTokenAllowed: 4096,
+    },
+    {
+      name: 'anthropic.claude-3-sonnet-20240229-v1:0',
+      label: 'Claude 3 Sonnet (Bedrock)',
+      provider: 'AmazonBedrock',
+      maxTokenAllowed: 4096,
+    },
+    {
+      name: 'anthropic.claude-3-haiku-20240307-v1:0',
+      label: 'Claude 3 Haiku (Bedrock)',
+      provider: 'AmazonBedrock',
+      maxTokenAllowed: 4096,
+    },
+    {
+      name: 'amazon.nova-pro-v1:0',
+      label: 'Amazon Nova Pro (Bedrock)',
+      provider: 'AmazonBedrock',
+      maxTokenAllowed: 5120,
+    },
+    {
+      name: 'amazon.nova-lite-v1:0',
+      label: 'Amazon Nova Lite (Bedrock)',
+      provider: 'AmazonBedrock',
+      maxTokenAllowed: 5120,
+    },
+    {
+      name: 'mistral.mistral-large-2402-v1:0',
+      label: 'Mistral Large 24.02 (Bedrock)',
+      provider: 'AmazonBedrock',
+      maxTokenAllowed: 8192,
+    },
+  ];
+
+  private _parseAndValidateConfig(apiKey: string): AWSBedRockConfig {
+    let parsedConfig: AWSBedRockConfig;
+
+    try {
+      parsedConfig = JSON.parse(apiKey);
+    } catch {
+      throw new Error(
+        'Invalid AWS Bedrock configuration format. Please provide a valid JSON string containing region, accessKeyId, and secretAccessKey.',
+      );
+    }
+
+    const { region, accessKeyId, secretAccessKey, sessionToken } = parsedConfig;
+
+    if (!region || !accessKeyId || !secretAccessKey) {
+      throw new Error(
+        'Missing required AWS credentials. Configuration must include region, accessKeyId, and secretAccessKey.',
+      );
+    }
+
+    return {
+      region,
+      accessKeyId,
+      secretAccessKey,
+      ...(sessionToken && { sessionToken }),
+    };
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const config = this._parseAndValidateConfig(apiKey);
+    const bedrock = createAmazonBedrock(config);
+
+    return bedrock(model);
+  }
+}

app/.server/modules/llm/providers/anthropic.ts

@@ -0,0 +1,78 @@
+import { createAnthropic } from '@ai-sdk/anthropic';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class AnthropicProvider extends BaseProvider {
+  name = 'Anthropic';
+  getApiKeyLink = 'https://console.anthropic.com/settings/keys';
+
+  staticModels: ModelInfo[] = [
+    {
+      name: 'claude-3-7-sonnet-20250219',
+      label: 'Claude 3.7 Sonnet',
+      provider: 'Anthropic',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'claude-3-5-sonnet-latest',
+      label: 'Claude 3.5 Sonnet (new)',
+      provider: 'Anthropic',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'claude-3-5-sonnet-20240620',
+      label: 'Claude 3.5 Sonnet (old)',
+      provider: 'Anthropic',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'claude-3-5-haiku-latest',
+      label: 'Claude 3.5 Haiku (new)',
+      provider: 'Anthropic',
+      maxTokenAllowed: 8000,
+    },
+    { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 },
+    { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 },
+    { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 },
+  ];
+
+  async getDynamicModels(settings: IProviderSetting): Promise<ModelInfo[]> {
+    const { apiKey } = this.getProviderBaseUrlAndKey(settings);
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const response = await fetch(`https://api.anthropic.com/v1/models`, {
+      headers: {
+        'x-api-key': `${apiKey}`,
+        'anthropic-version': '2023-06-01',
+      },
+    });
+
+    const res = (await response.json()) as any;
+    const staticModelIds = this.staticModels.map((m) => m.name);
+
+    const data = res.data.filter((model: any) => model.type === 'model' && !staticModelIds.includes(model.id));
+
+    return data.map((m: any) => ({
+      name: m.id,
+      label: `${m.display_name}`,
+      provider: this.name,
+      maxTokenAllowed: 32000,
+    }));
+  }
+
+  getModelInstance: (options: { model: string; providerSettings?: Record<string, IProviderSetting> }) => LanguageModel =
+    (options) => {
+      const { providerSettings, model } = options;
+      const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings);
+      const anthropic = createAnthropic({
+        apiKey,
+      });
+
+      return anthropic(model);
+    };
+}

app/.server/modules/llm/providers/cohere.ts

@@ -0,0 +1,39 @@
+import { createCohere } from '@ai-sdk/cohere';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class CohereProvider extends BaseProvider {
+  name = 'Cohere';
+  getApiKeyLink = 'https://dashboard.cohere.com/api-keys';
+
+  staticModels: ModelInfo[] = [
+    { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 },
+    { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 },
+    { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 },
+    { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 },
+    { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 },
+    { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 },
+    { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 },
+    { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 },
+    { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 },
+    { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 },
+  ];
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const cohere = createCohere({
+      apiKey,
+    });
+
+    return cohere(model);
+  }
+}

app/.server/modules/llm/providers/deepseek.ts

@@ -0,0 +1,37 @@
+import { createDeepSeek } from '@ai-sdk/deepseek';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class DeepseekProvider extends BaseProvider {
+  name = 'DeepSeek';
+  getApiKeyLink = 'https://platform.deepseek.com/apiKeys';
+
+  config = {
+    apiTokenKey: 'DEEPSEEK_API_KEY',
+    baseUrlKey: '',
+  };
+
+  staticModels: ModelInfo[] = [
+    { name: 'deepseek-coder', label: 'DeepSeek-Coder', provider: 'DeepSeek', maxTokenAllowed: 8000 },
+    { name: 'deepseek-chat', label: 'DeepSeek-Chat', provider: 'DeepSeek', maxTokenAllowed: 8000 },
+    { name: 'deepseek-reasoner', label: 'DeepSeek-Reasoner', provider: 'DeepSeek', maxTokenAllowed: 8000 },
+  ];
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const deepseek = createDeepSeek({
+      apiKey,
+    });
+
+    return deepseek(model);
+  }
+}

app/.server/modules/llm/providers/doubao.ts

@@ -0,0 +1,57 @@
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class DouBaoProvider extends BaseProvider {
+  name = 'DouBao';
+  getApiKeyLink = undefined;
+
+  staticModels: ModelInfo[] = [];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    const { baseUrl: fetchBaseUrl, apiKey } = this.getProviderBaseUrlAndKey(settings);
+    const baseUrl = fetchBaseUrl || 'https://ark.cn-beijing.volces.com/api/v3';
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const response = await fetch(`${baseUrl}/models`, {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    });
+
+    const res = (await response.json()) as any;
+
+    const data = res.data.filter((model: any) => model.object === 'model' && model.supports_chat);
+
+    return data.map((m: any) => ({
+      name: m.id,
+      label: `${m.id} - context ${m.context_length ? Math.floor(m.context_length / 1000) + 'k' : 'N/A'}`,
+      provider: this.name,
+      maxTokenAllowed: m.context_length || 8000,
+    }));
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const provider = createOpenAICompatible({
+      name: this.name,
+      baseURL: 'https://ark.cn-beijing.volces.com/api/v3',
+      apiKey,
+      includeUsage: true,
+    });
+
+    return provider(model);
+  }
+}

app/.server/modules/llm/providers/ernie.ts

@@ -0,0 +1,57 @@
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class ErnieProvider extends BaseProvider {
+  name = 'Ernie';
+  getApiKeyLink = undefined;
+
+  staticModels: ModelInfo[] = [];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    const { baseUrl: fetchBaseUrl, apiKey } = this.getProviderBaseUrlAndKey(settings);
+    const baseUrl = fetchBaseUrl || 'https://qianfan.baidubce.com/v2';
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const response = await fetch(`${baseUrl}/models`, {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    });
+
+    const res = (await response.json()) as any;
+
+    const data = res.data.filter((model: any) => model.object === 'model' && model.supports_chat);
+
+    return data.map((m: any) => ({
+      name: m.id,
+      label: `${m.id} - context ${m.context_length ? Math.floor(m.context_length / 1000) + 'k' : 'N/A'}`,
+      provider: this.name,
+      maxTokenAllowed: m.context_length || 8000,
+    }));
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const provider = createOpenAICompatible({
+      name: this.name,
+      baseURL: 'https://qianfan.baidubce.com/v2',
+      apiKey,
+      includeUsage: true,
+    });
+
+    return provider(model);
+  }
+}

app/.server/modules/llm/providers/github.ts

@@ -0,0 +1,38 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class GithubProvider extends BaseProvider {
+  name = 'Github';
+  getApiKeyLink = 'https://github.com/settings/personal-access-tokens';
+
+  // find more in https://github.com/marketplace?type=models
+  staticModels: ModelInfo[] = [
+    { name: 'gpt-4o', label: 'GPT-4o', provider: 'Github', maxTokenAllowed: 8000 },
+    { name: 'o1', label: 'o1-preview', provider: 'Github', maxTokenAllowed: 100000 },
+    { name: 'o1-mini', label: 'o1-mini', provider: 'Github', maxTokenAllowed: 8000 },
+    { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'Github', maxTokenAllowed: 8000 },
+    { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'Github', maxTokenAllowed: 8000 },
+    { name: 'gpt-4', label: 'GPT-4', provider: 'Github', maxTokenAllowed: 8000 },
+    { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'Github', maxTokenAllowed: 8000 },
+  ];
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const openai = createOpenAI({
+      baseURL: 'https://models.inference.ai.azure.com',
+      apiKey,
+    });
+
+    return openai(model);
+  }
+}

app/.server/modules/llm/providers/google.ts

@@ -0,0 +1,68 @@
+import { createGoogleGenerativeAI } from '@ai-sdk/google';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class GoogleProvider extends BaseProvider {
+  name = 'Google';
+  getApiKeyLink = 'https://aistudio.google.com/app/apikey';
+
+  staticModels: ModelInfo[] = [
+    { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 },
+    {
+      name: 'gemini-2.0-flash-thinking-exp-01-21',
+      label: 'Gemini 2.0 Flash-thinking-exp-01-21',
+      provider: 'Google',
+      maxTokenAllowed: 65536,
+    },
+    { name: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash', provider: 'Google', maxTokenAllowed: 8192 },
+    { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 },
+    { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 },
+    { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 },
+    { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 },
+    { name: 'gemini-exp-1206', label: 'Gemini exp-1206', provider: 'Google', maxTokenAllowed: 8192 },
+  ];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    const { apiKey } = this.getProviderBaseUrlAndKey(settings);
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models?key=${apiKey}`, {
+      headers: {
+        // biome-ignore lint: ignore Content-Type header
+        ['Content-Type']: 'application/json',
+      },
+    });
+
+    const res = (await response.json()) as any;
+
+    const data = res.models.filter((model: any) => model.outputTokenLimit > 8000);
+
+    return data.map((m: any) => ({
+      name: m.name.replace('models/', ''),
+      label: `${m.displayName} - context ${Math.floor((m.inputTokenLimit + m.outputTokenLimit) / 1000) + 'k'}`,
+      provider: this.name,
+      maxTokenAllowed: m.inputTokenLimit + m.outputTokenLimit || 8000,
+    }));
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const google = createGoogleGenerativeAI({
+      apiKey,
+    });
+
+    return google(model);
+  }
+}

app/.server/modules/llm/providers/groq.ts

@@ -0,0 +1,69 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class GroqProvider extends BaseProvider {
+  name = 'Groq';
+  getApiKeyLink = 'https://console.groq.com/keys';
+
+  staticModels: ModelInfo[] = [
+    { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+    { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+    { name: 'llama-3.2-90b-vision-preview', label: 'Llama 3.2 90b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+    { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+    { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+    { name: 'llama-3.3-70b-versatile', label: 'Llama 3.3 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+    {
+      name: 'deepseek-r1-distill-llama-70b',
+      label: 'DeepSeek R1 Distill Llama 70b (Groq)',
+      provider: 'Groq',
+      maxTokenAllowed: 131072,
+    },
+  ];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    const { apiKey } = this.getProviderBaseUrlAndKey(settings);
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const response = await fetch(`https://api.groq.com/openai/v1/models`, {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    });
+
+    const res = (await response.json()) as any;
+
+    const data = res.data.filter(
+      (model: any) => model.object === 'model' && model.active && model.context_window > 8000,
+    );
+
+    return data.map((m: any) => ({
+      name: m.id,
+      label: `${m.id} - context ${m.context_window ? Math.floor(m.context_window / 1000) + 'k' : 'N/A'} [ by ${m.owned_by}]`,
+      provider: this.name,
+      maxTokenAllowed: m.context_window || 8000,
+    }));
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const openai = createOpenAI({
+      baseURL: 'https://api.groq.com/openai/v1',
+      apiKey,
+    });
+
+    return openai(model);
+  }
+}

app/.server/modules/llm/providers/huggingface.ts

@@ -0,0 +1,96 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class HuggingFaceProvider extends BaseProvider {
+  name = 'HuggingFace';
+  getApiKeyLink = 'https://huggingface.co/settings/tokens';
+
+  staticModels: ModelInfo[] = [
+    {
+      name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+      label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: '01-ai/Yi-1.5-34B-Chat',
+      label: 'Yi-1.5-34B-Chat (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'codellama/CodeLlama-34b-Instruct-hf',
+      label: 'CodeLlama-34b-Instruct (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'NousResearch/Hermes-3-Llama-3.1-8B',
+      label: 'Hermes-3-Llama-3.1-8B (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+      label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'Qwen/Qwen2.5-72B-Instruct',
+      label: 'Qwen2.5-72B-Instruct (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'meta-llama/Llama-3.1-70B-Instruct',
+      label: 'Llama-3.1-70B-Instruct (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'meta-llama/Llama-3.1-405B',
+      label: 'Llama-3.1-405B (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: '01-ai/Yi-1.5-34B-Chat',
+      label: 'Yi-1.5-34B-Chat (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'codellama/CodeLlama-34b-Instruct-hf',
+      label: 'CodeLlama-34b-Instruct (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'NousResearch/Hermes-3-Llama-3.1-8B',
+      label: 'Hermes-3-Llama-3.1-8B (HuggingFace)',
+      provider: 'HuggingFace',
+      maxTokenAllowed: 8000,
+    },
+  ];
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const openai = createOpenAI({
+      baseURL: 'https://api-inference.huggingface.co/v1/',
+      apiKey,
+    });
+
+    return openai(model);
+  }
+}

app/.server/modules/llm/providers/hyperbolic.ts

@@ -0,0 +1,86 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class HyperbolicProvider extends BaseProvider {
+  name = 'Hyperbolic';
+  getApiKeyLink = 'https://app.hyperbolic.xyz/settings';
+
+  staticModels: ModelInfo[] = [
+    {
+      name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+      label: 'Qwen 2.5 Coder 32B Instruct',
+      provider: 'Hyperbolic',
+      maxTokenAllowed: 8192,
+    },
+    {
+      name: 'Qwen/Qwen2.5-72B-Instruct',
+      label: 'Qwen2.5-72B-Instruct',
+      provider: 'Hyperbolic',
+      maxTokenAllowed: 8192,
+    },
+    {
+      name: 'deepseek-ai/DeepSeek-V2.5',
+      label: 'DeepSeek-V2.5',
+      provider: 'Hyperbolic',
+      maxTokenAllowed: 8192,
+    },
+    {
+      name: 'Qwen/QwQ-32B-Preview',
+      label: 'QwQ-32B-Preview',
+      provider: 'Hyperbolic',
+      maxTokenAllowed: 8192,
+    },
+    {
+      name: 'Qwen/Qwen2-VL-72B-Instruct',
+      label: 'Qwen2-VL-72B-Instruct',
+      provider: 'Hyperbolic',
+      maxTokenAllowed: 8192,
+    },
+  ];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    const { baseUrl: fetchBaseUrl, apiKey } = this.getProviderBaseUrlAndKey(settings);
+    const baseUrl = fetchBaseUrl || 'https://api.hyperbolic.xyz/v1';
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const response = await fetch(`${baseUrl}/models`, {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    });
+
+    const res = (await response.json()) as any;
+
+    const data = res.data.filter((model: any) => model.object === 'model' && model.supports_chat);
+
+    return data.map((m: any) => ({
+      name: m.id,
+      label: `${m.id} - context ${m.context_length ? Math.floor(m.context_length / 1000) + 'k' : 'N/A'}`,
+      provider: this.name,
+      maxTokenAllowed: m.context_length || 8000,
+    }));
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const openai = createOpenAI({
+      baseURL: 'https://api.hyperbolic.xyz/v1/',
+      apiKey,
+    });
+
+    return openai(model);
+  }
+}

app/.server/modules/llm/providers/kimi.ts

@@ -0,0 +1,57 @@
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class KimiProvider extends BaseProvider {
+  name = 'Kimi';
+  getApiKeyLink = undefined;
+
+  staticModels: ModelInfo[] = [];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    const { baseUrl: fetchBaseUrl, apiKey } = this.getProviderBaseUrlAndKey(settings);
+    const baseUrl = fetchBaseUrl || 'https://api.moonshot.cn/v1';
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const response = await fetch(`${baseUrl}/models`, {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    });
+
+    const res = (await response.json()) as any;
+
+    const data = res.data.filter((model: any) => model.object === 'model' && model.supports_chat);
+
+    return data.map((m: any) => ({
+      name: m.id,
+      label: `${m.id} - context ${m.context_length ? Math.floor(m.context_length / 1000) + 'k' : 'N/A'}`,
+      provider: this.name,
+      maxTokenAllowed: m.context_length || 8000,
+    }));
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const provider = createOpenAICompatible({
+      name: this.name,
+      baseURL: 'https://api.moonshot.cn/v1',
+      apiKey,
+      includeUsage: true,
+    });
+
+    return provider(model);
+  }
+}

app/.server/modules/llm/providers/lmstudio.ts

@@ -0,0 +1,72 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import { logger } from '~/utils/logger';
+
+export const BASE_URL = 'http://127.0.0.1:1234/';
+export default class LMStudioProvider extends BaseProvider {
+  name = 'LMStudio';
+  getApiKeyLink = 'https://lmstudio.ai/';
+  labelForGetApiKey = 'Get LMStudio';
+  icon = 'i-ph:cloud-arrow-down';
+
+  staticModels: ModelInfo[] = [];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    let { baseUrl } = this.getProviderBaseUrlAndKey(settings);
+
+    if (!baseUrl) {
+      logger.debug('No baseUrl found for LMStudio provider, using default: ', BASE_URL);
+      baseUrl = BASE_URL;
+    }
+
+    if (typeof window === 'undefined') {
+      /*
+       * Running in Server
+       * Backend: Check if we're running in Docker
+       */
+      const isDocker = process?.env?.RUNNING_IN_DOCKER === 'true';
+
+      baseUrl = isDocker ? baseUrl.replace('localhost', 'host.docker.internal') : baseUrl;
+      baseUrl = isDocker ? baseUrl.replace('127.0.0.1', 'host.docker.internal') : baseUrl;
+    }
+
+    const response = await fetch(`${baseUrl}/v1/models`);
+    const data = (await response.json()) as { data: Array<{ id: string }> };
+
+    return data.data.map((model) => ({
+      name: model.id,
+      label: model.id,
+      provider: this.name,
+      maxTokenAllowed: 8000,
+    }));
+  }
+  getModelInstance: (options: { model: string; providerSettings?: Record<string, IProviderSetting> }) => LanguageModel =
+    (options) => {
+      const { providerSettings, model } = options;
+      let { baseUrl } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+      if (!baseUrl) {
+        logger.debug('No baseUrl found for LMStudio provider, using default: ', BASE_URL);
+        baseUrl = BASE_URL;
+      }
+
+      const isDocker = process?.env?.RUNNING_IN_DOCKER === 'true';
+
+      if (typeof window === 'undefined') {
+        baseUrl = isDocker ? baseUrl.replace('localhost', 'host.docker.internal') : baseUrl;
+        baseUrl = isDocker ? baseUrl.replace('127.0.0.1', 'host.docker.internal') : baseUrl;
+      }
+
+      logger.debug('LMStudio Base Url used: ', baseUrl);
+
+      const lmstudio = createOpenAI({
+        baseURL: `${baseUrl}/v1`,
+        apiKey: '',
+      });
+
+      return lmstudio(model);
+    };
+}

app/.server/modules/llm/providers/mistral.ts

@@ -0,0 +1,38 @@
+import { createMistral } from '@ai-sdk/mistral';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class MistralProvider extends BaseProvider {
+  name = 'Mistral';
+  getApiKeyLink = 'https://console.mistral.ai/api-keys/';
+
+  staticModels: ModelInfo[] = [
+    { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 },
+    { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 },
+    { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 },
+    { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 },
+    { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 },
+    { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 },
+    { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 },
+    { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 },
+    { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 },
+  ];
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const mistral = createMistral({
+      apiKey,
+    });
+
+    return mistral(model);
+  }
+}

app/.server/modules/llm/providers/ollama.ts

@@ -0,0 +1,100 @@
+import type { LanguageModel } from 'ai';
+import { createOllama } from 'ollama-ai-provider-v2';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import { logger } from '~/utils/logger';
+
+interface OllamaModelDetails {
+  parent_model: string;
+  format: string;
+  family: string;
+  families: string[];
+  parameter_size: string;
+  quantization_level: string;
+}
+
+export interface OllamaModel {
+  name: string;
+  model: string;
+  modified_at: string;
+  size: number;
+  digest: string;
+  details: OllamaModelDetails;
+}
+
+export interface OllamaApiResponse {
+  models: OllamaModel[];
+}
+
+const BASE_URL = 'http://127.0.0.1:11434';
+
+export default class OllamaProvider extends BaseProvider {
+  name = 'Ollama';
+  getApiKeyLink = 'https://ollama.com/download';
+  labelForGetApiKey = 'Download Ollama';
+  icon = 'i-ph:cloud-arrow-down';
+
+  staticModels: ModelInfo[] = [];
+
+  getDefaultNumCtx(): number {
+    return process.env.DEFAULT_NUM_CTX ? parseInt(process.env.DEFAULT_NUM_CTX, 10) : 32768;
+  }
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    let { baseUrl } = this.getProviderBaseUrlAndKey(settings);
+
+    if (!baseUrl) {
+      logger.debug('No baseUrl found for OLLAMA provider, using default: ', BASE_URL);
+      baseUrl = BASE_URL;
+    }
+
+    if (typeof window === 'undefined') {
+      /*
+       * Running in Server
+       * Backend: Check if we're running in Docker
+       */
+      const isDocker = process?.env?.RUNNING_IN_DOCKER === 'true';
+
+      baseUrl = isDocker ? baseUrl.replace('localhost', 'host.docker.internal') : baseUrl;
+      baseUrl = isDocker ? baseUrl.replace('127.0.0.1', 'host.docker.internal') : baseUrl;
+    }
+
+    const response = await fetch(`${baseUrl}/api/tags`);
+    const data = (await response.json()) as OllamaApiResponse;
+
+    // console.log({ ollamamodels: data.models });
+
+    return data.models.map((model: OllamaModel) => ({
+      name: model.name,
+      label: `${model.name} (${model.details.parameter_size})`,
+      provider: this.name,
+      maxTokenAllowed: 8000,
+    }));
+  }
+
+  getModelInstance: (options: { model: string; providerSettings?: Record<string, IProviderSetting> }) => LanguageModel =
+    (options) => {
+      const { providerSettings, model } = options;
+
+      let { baseUrl } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+      // Backend: Check if we're running in Docker
+      if (!baseUrl) {
+        logger.debug('No baseUrl found for OLLAMA provider, using default: ', BASE_URL);
+        baseUrl = BASE_URL;
+      }
+
+      const isDocker = process?.env?.RUNNING_IN_DOCKER === 'true';
+      baseUrl = isDocker ? baseUrl.replace('localhost', 'host.docker.internal') : baseUrl;
+      baseUrl = isDocker ? baseUrl.replace('127.0.0.1', 'host.docker.internal') : baseUrl;
+
+      logger.debug('Ollama Base Url used: ', baseUrl);
+
+      const ollama = createOllama({
+        baseURL: `${baseUrl}/api`,
+      });
+
+      return ollama(model);
+    };
+}

app/.server/modules/llm/providers/open-router.ts

@@ -0,0 +1,111 @@
+import { createOpenRouter } from '@openrouter/ai-sdk-provider';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+interface OpenRouterModel {
+  name: string;
+  id: string;
+  context_length: number;
+  pricing: {
+    prompt: number;
+    completion: number;
+  };
+}
+
+interface OpenRouterModelsResponse {
+  data: OpenRouterModel[];
+}
+
+export default class OpenRouterProvider extends BaseProvider {
+  name = 'OpenRouter';
+  getApiKeyLink = 'https://openrouter.ai/settings/keys';
+
+  staticModels: ModelInfo[] = [
+    {
+      name: 'anthropic/claude-3.5-sonnet',
+      label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)',
+      provider: 'OpenRouter',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'anthropic/claude-3-haiku',
+      label: 'Anthropic: Claude 3 Haiku (OpenRouter)',
+      provider: 'OpenRouter',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'deepseek/deepseek-coder',
+      label: 'DeepSeek-Coder V2 236B (OpenRouter)',
+      provider: 'OpenRouter',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'google/gemini-flash-1.5',
+      label: 'Google Gemini Flash 1.5 (OpenRouter)',
+      provider: 'OpenRouter',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'google/gemini-pro-1.5',
+      label: 'Google Gemini Pro 1.5 (OpenRouter)',
+      provider: 'OpenRouter',
+      maxTokenAllowed: 8000,
+    },
+    { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
+    {
+      name: 'mistralai/mistral-nemo',
+      label: 'OpenRouter Mistral Nemo (OpenRouter)',
+      provider: 'OpenRouter',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'qwen/qwen-110b-chat',
+      label: 'OpenRouter Qwen 110b Chat (OpenRouter)',
+      provider: 'OpenRouter',
+      maxTokenAllowed: 8000,
+    },
+    { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 },
+  ];
+
+  async getDynamicModels(_settings?: IProviderSetting, _serverEnv: Record<string, string> = {}): Promise<ModelInfo[]> {
+    try {
+      const response = await fetch('https://openrouter.ai/api/v1/models', {
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      });
+
+      const data = (await response.json()) as OpenRouterModelsResponse;
+
+      return data.data
+        .sort((a, b) => a.name.localeCompare(b.name))
+        .map((m) => ({
+          name: m.id,
+          label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed(2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`,
+          provider: this.name,
+          maxTokenAllowed: 8000,
+        }));
+    } catch (error) {
+      console.error('Error getting OpenRouter models:', error);
+      return [];
+    }
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const openRouter = createOpenRouter({
+      apiKey,
+    });
+
+    return openRouter.chat(model);
+  }
+}

app/.server/modules/llm/providers/openai.ts

@@ -0,0 +1,64 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class OpenAILikeProvider extends BaseProvider {
+  name = 'OpenAI';
+  getApiKeyLink = undefined;
+
+  staticModels: ModelInfo[] = [];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    const { baseUrl, apiKey } = this.getProviderBaseUrlAndKey(settings);
+
+    if (!baseUrl || !apiKey) {
+      return [];
+    }
+
+    const response = await fetch(`${baseUrl}/models`, {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    });
+
+    const res = (await response.json()) as any;
+
+    return res.data.map((model: any) => ({
+      name: model.id,
+      label: model.id,
+      provider: this.name,
+      maxTokenAllowed: 8000,
+    }));
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { baseUrl, apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing configuration for ${this.name} provider`);
+    }
+
+    if (!!baseUrl) {
+      const provider = createOpenAICompatible({
+        name: this.name,
+        baseURL: baseUrl,
+        apiKey,
+        includeUsage: true,
+      });
+
+      return provider(model);
+    }
+
+    const openai = createOpenAI({
+      baseURL: baseUrl,
+      apiKey,
+    });
+
+    return openai(model);
+  }
+}

app/.server/modules/llm/providers/perplexity.ts

@@ -0,0 +1,48 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class PerplexityProvider extends BaseProvider {
+  name = 'Perplexity';
+  getApiKeyLink = 'https://www.perplexity.ai/settings/api';
+
+  staticModels: ModelInfo[] = [
+    {
+      name: 'llama-3.1-sonar-small-128k-online',
+      label: 'Sonar Small Online',
+      provider: 'Perplexity',
+      maxTokenAllowed: 8192,
+    },
+    {
+      name: 'llama-3.1-sonar-large-128k-online',
+      label: 'Sonar Large Online',
+      provider: 'Perplexity',
+      maxTokenAllowed: 8192,
+    },
+    {
+      name: 'llama-3.1-sonar-huge-128k-online',
+      label: 'Sonar Huge Online',
+      provider: 'Perplexity',
+      maxTokenAllowed: 8192,
+    },
+  ];
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const perplexity = createOpenAI({
+      baseURL: 'https://api.perplexity.ai/',
+      apiKey,
+    });
+
+    return perplexity(model);
+  }
+}

app/.server/modules/llm/providers/qwen.ts

@@ -0,0 +1,57 @@
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class QwenProvider extends BaseProvider {
+  name = 'Qwen';
+  getApiKeyLink = undefined;
+
+  staticModels: ModelInfo[] = [];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    const { baseUrl: fetchBaseUrl, apiKey } = this.getProviderBaseUrlAndKey(settings);
+    const baseUrl = fetchBaseUrl || 'https://dashscope.aliyuncs.com/compatible-mode/v1';
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const response = await fetch(`${baseUrl}/models`, {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    });
+
+    const res = (await response.json()) as any;
+
+    const data = res.data.filter((model: any) => model.object === 'model' && model.supports_chat);
+
+    return data.map((m: any) => ({
+      name: m.id,
+      label: `${m.id} - context ${m.context_length ? Math.floor(m.context_length / 1000) + 'k' : 'N/A'}`,
+      provider: this.name,
+      maxTokenAllowed: m.context_length || 8000,
+    }));
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const provider = createOpenAICompatible({
+      name: this.name,
+      baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+      apiKey,
+      includeUsage: true,
+    });
+
+    return provider(model);
+  }
+}

app/.server/modules/llm/providers/together.ts

@@ -0,0 +1,70 @@
+import type { LanguageModel } from 'ai';
+import { BaseProvider, getOpenAILikeModel } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class TogetherProvider extends BaseProvider {
+  name = 'Together';
+  getApiKeyLink = 'https://api.together.xyz/settings/api-keys';
+
+  staticModels: ModelInfo[] = [
+    {
+      name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+      label: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+      provider: 'Together',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo',
+      label: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo',
+      provider: 'Together',
+      maxTokenAllowed: 8000,
+    },
+    {
+      name: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
+      label: 'Mixtral 8x7B Instruct',
+      provider: 'Together',
+      maxTokenAllowed: 8192,
+    },
+  ];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    const { baseUrl: fetchBaseUrl, apiKey } = this.getProviderBaseUrlAndKey(settings);
+    const baseUrl = fetchBaseUrl || 'https://api.together.xyz/v1';
+
+    if (!baseUrl || !apiKey) {
+      return [];
+    }
+
+    // console.log({ baseUrl, apiKey });
+
+    const response = await fetch(`${baseUrl}/models`, {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    });
+
+    const res = (await response.json()) as any;
+    const data = (res || []).filter((model: any) => model.type === 'chat');
+
+    return data.map((m: any) => ({
+      name: m.id,
+      label: `${m.display_name} - in:$${m.pricing.input.toFixed(2)} out:$${m.pricing.output.toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`,
+      provider: this.name,
+      maxTokenAllowed: 8000,
+    }));
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { baseUrl: fetchBaseUrl, apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+    const baseUrl = fetchBaseUrl || 'https://api.together.xyz/v1';
+
+    if (!apiKey) {
+      throw new Error(`Missing configuration for ${this.name} provider`);
+    }
+
+    return getOpenAILikeModel(baseUrl, apiKey, model);
+  }
+}

app/.server/modules/llm/providers/xai.ts

@@ -0,0 +1,32 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class XAIProvider extends BaseProvider {
+  name = 'xAI';
+  getApiKeyLink = 'https://docs.x.ai/docs/quickstart#creating-an-api-key';
+
+  staticModels: ModelInfo[] = [
+    { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 },
+    { name: 'grok-2-1212', label: 'xAI Grok2 1212', provider: 'xAI', maxTokenAllowed: 8000 },
+  ];
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw new Error(`Missing API key for ${this.name} provider`);
+    }
+
+    const openai = createOpenAI({
+      baseURL: 'https://api.x.ai/v1',
+      apiKey,
+    });
+
+    return openai(model);
+  }
+}

app/.server/modules/llm/providers/zhipu.ts

@@ -0,0 +1,57 @@
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+import type { LanguageModel } from 'ai';
+import { BaseProvider } from '~/.server/modules/llm/base-provider';
+import type { ModelInfo } from '~/.server/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+
+export default class ZhiPuProvider extends BaseProvider {
+  name = 'ZhiPu';
+  getApiKeyLink = undefined;
+
+  staticModels: ModelInfo[] = [];
+
+  async getDynamicModels(settings?: IProviderSetting): Promise<ModelInfo[]> {
+    const { baseUrl: fetchBaseUrl, apiKey } = this.getProviderBaseUrlAndKey(settings);
+    const baseUrl = fetchBaseUrl || 'https://open.bigmodel.cn/api/paas/v4';
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const response = await fetch(`${baseUrl}/models`, {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    });
+
+    const res = (await response.json()) as any;
+
+    const data = res.data.filter((model: any) => model.object === 'model' && model.supports_chat);
+
+    return data.map((m: any) => ({
+      name: m.id,
+      label: `${m.id} - context ${m.context_length ? Math.floor(m.context_length / 1000) + 'k' : 'N/A'}`,
+      provider: this.name,
+      maxTokenAllowed: m.context_length || 8000,
+    }));
+  }
+
+  getModelInstance(options: { model: string; providerSettings?: Record<string, IProviderSetting> }): LanguageModel {
+    const { model, providerSettings } = options;
+
+    const { apiKey } = this.getProviderBaseUrlAndKey(providerSettings?.[this.name]);
+
+    if (!apiKey) {
+      throw `Missing Api Key configuration for ${this.name} provider`;
+    }
+
+    const openai = createOpenAICompatible({
+      name: this.name,
+      baseURL: 'https://open.bigmodel.cn/api/paas/v4',
+      apiKey,
+      includeUsage: true,
+    });
+
+    return openai(model);
+  }
+}

app/.server/modules/llm/registry.ts

@@ -0,0 +1,47 @@
+import AmazonBedrockProvider from './providers/amazon-bedrock';
+import AnthropicProvider from './providers/anthropic';
+import CohereProvider from './providers/cohere';
+import DeepseekProvider from './providers/deepseek';
+import DouBaoProvider from './providers/doubao';
+import ErnieProvider from './providers/ernie';
+import GithubProvider from './providers/github';
+import GoogleProvider from './providers/google';
+import GroqProvider from './providers/groq';
+import HuggingFaceProvider from './providers/huggingface';
+import HyperbolicProvider from './providers/hyperbolic';
+import KimiProvider from './providers/kimi';
+import LMStudioProvider from './providers/lmstudio';
+import MistralProvider from './providers/mistral';
+import OllamaProvider from './providers/ollama';
+import OpenRouterProvider from './providers/open-router';
+import OpenAIProvider from './providers/openai';
+import PerplexityProvider from './providers/perplexity';
+import QwenProvider from './providers/qwen';
+import TogetherProvider from './providers/together';
+import XAIProvider from './providers/xai';
+import ZhiPuProvider from './providers/zhipu';
+
+export {
+  AnthropicProvider,
+  CohereProvider,
+  DeepseekProvider,
+  DouBaoProvider,
+  ErnieProvider,
+  KimiProvider,
+  QwenProvider,
+  ZhiPuProvider,
+  GoogleProvider,
+  GroqProvider,
+  HuggingFaceProvider,
+  HyperbolicProvider,
+  MistralProvider,
+  OllamaProvider,
+  OpenAIProvider,
+  OpenRouterProvider,
+  PerplexityProvider,
+  XAIProvider,
+  TogetherProvider,
+  LMStudioProvider,
+  AmazonBedrockProvider,
+  GithubProvider,
+};

app/.server/modules/llm/types.ts

@@ -0,0 +1,27 @@
+import type { LanguageModel } from 'ai';
+import type { IProviderSetting } from '~/types/model';
+
+export interface ModelInfo {
+  name: string;
+  label: string;
+  provider: string;
+  maxTokenAllowed: number;
+}
+
+export interface ProviderInfo {
+  name: string;
+  staticModels: ModelInfo[];
+  getDynamicModels?: (apiKeys?: Record<string, string>, settings?: IProviderSetting) => Promise<ModelInfo[]>;
+  getModelInstance: (options: {
+    model: string;
+    apiKeys?: Record<string, string>;
+    providerSettings?: Record<string, IProviderSetting>;
+  }) => LanguageModel;
+  getApiKeyLink?: string;
+  labelForGetApiKey?: string;
+  icon?: string;
+}
+export interface ProviderConfig {
+  baseUrlKey?: string;
+  apiTokenKey?: string;
+}