From de0084c8df777390704c21670dde216085ef7101 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Thu, 22 Feb 2024 03:22:23 -0800 Subject: [PATCH 01/14] feat: litellm integration --- backend/main.py | 4 ++++ backend/requirements.txt | 2 ++ 2 files changed, 6 insertions(+) diff --git a/backend/main.py b/backend/main.py index d1fb0c20..2ec741b0 100644 --- a/backend/main.py +++ b/backend/main.py @@ -8,6 +8,8 @@ from fastapi.middleware.cors import CORSMiddleware from starlette.exceptions import HTTPException as StarletteHTTPException +from litellm.proxy.proxy_server import app as litellm_app + from apps.ollama.main import app as ollama_app from apps.openai.main import app as openai_app from apps.audio.main import app as audio_app @@ -54,6 +56,8 @@ async def check_url(request: Request, call_next): app.mount("/api/v1", webui_app) +app.mount("/litellm/api", litellm_app) + app.mount("/ollama/api", ollama_app) app.mount("/openai/api", openai_app) diff --git a/backend/requirements.txt b/backend/requirements.txt index 56e1d36e..75d368d8 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -16,6 +16,8 @@ aiohttp peewee bcrypt +litellm[proxy] + langchain langchain-community chromadb From 9b6dca3d7f59d853acbeb7a00f57c925ac111cc4 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Thu, 22 Feb 2024 04:12:26 -0800 Subject: [PATCH 02/14] feat: litellm frontend integration --- src/lib/apis/litellm/index.ts | 42 +++++++ src/lib/apis/ollama/index.ts | 8 +- src/lib/apis/openai/index.ts | 12 +- src/lib/components/chat/ModelSelector.svelte | 4 +- src/lib/components/chat/SettingsModal.svelte | 9 ++ src/lib/constants.ts | 2 + src/routes/(app)/+layout.svelte | 8 ++ src/routes/(app)/+page.svelte | 111 ++++++++++--------- 8 files changed, 134 insertions(+), 62 deletions(-) create mode 100644 src/lib/apis/litellm/index.ts diff --git a/src/lib/apis/litellm/index.ts b/src/lib/apis/litellm/index.ts new file mode 100644 index 00000000..87d2654e --- /dev/null +++ b/src/lib/apis/litellm/index.ts @@ -0,0 +1,42 @@ +import { LITELLM_API_BASE_URL } from '$lib/constants'; + +export const getLiteLLMModels = async (token: string = '') => { + let error = null; + + const res = await fetch(`${LITELLM_API_BASE_URL}/v1/models`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = `OpenAI: ${err?.error?.message ?? 'Network Problem'}`; + return []; + }); + + if (error) { + throw error; + } + + const models = Array.isArray(res) ? res : res?.data ?? null; + + return models + ? models + .map((model) => ({ + id: model.id, + name: model.name ?? model.id, + external: true, + source: 'litellm' + })) + .sort((a, b) => { + return a.name.localeCompare(b.name); + }) + : models; +}; diff --git a/src/lib/apis/ollama/index.ts b/src/lib/apis/ollama/index.ts index 954956ac..5fc8a5fe 100644 --- a/src/lib/apis/ollama/index.ts +++ b/src/lib/apis/ollama/index.ts @@ -128,9 +128,11 @@ export const getOllamaModels = async (token: string = '') => { throw error; } - return (res?.models ?? []).sort((a, b) => { - return a.name.localeCompare(b.name); - }); + return (res?.models ?? []) + .map((model) => ({ id: model.model, name: model.name ?? model.model, ...model })) + .sort((a, b) => { + return a.name.localeCompare(b.name); + }); }; // TODO: migrate to backend diff --git a/src/lib/apis/openai/index.ts b/src/lib/apis/openai/index.ts index f9173f70..41445581 100644 --- a/src/lib/apis/openai/index.ts +++ b/src/lib/apis/openai/index.ts @@ -163,7 +163,7 @@ export const getOpenAIModels = async (token: string = '') => { return models ? models - .map((model) => ({ name: model.id, external: true })) + .map((model) => ({ id: model.id, name: model.name ?? model.id, external: true })) .sort((a, b) => { return a.name.localeCompare(b.name); }) @@ -200,17 +200,21 @@ export const getOpenAIModelsDirect = async ( const models = Array.isArray(res) ? res : res?.data ?? null; return models - .map((model) => ({ name: model.id, external: true })) + .map((model) => ({ id: model.id, name: model.name ?? model.id, external: true })) .filter((model) => (base_url.includes('openai') ? model.name.includes('gpt') : true)) .sort((a, b) => { return a.name.localeCompare(b.name); }); }; -export const generateOpenAIChatCompletion = async (token: string = '', body: object) => { +export const generateOpenAIChatCompletion = async ( + token: string = '', + body: object, + url: string = OPENAI_API_BASE_URL +) => { let error = null; - const res = await fetch(`${OPENAI_API_BASE_URL}/chat/completions`, { + const res = await fetch(`${url}/chat/completions`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, diff --git a/src/lib/components/chat/ModelSelector.svelte b/src/lib/components/chat/ModelSelector.svelte index 81304cb4..8f478916 100644 --- a/src/lib/components/chat/ModelSelector.svelte +++ b/src/lib/components/chat/ModelSelector.svelte @@ -25,7 +25,7 @@ $: if (selectedModels.length > 0 && $models.length > 0) { selectedModels = selectedModels.map((model) => - $models.map((m) => m.name).includes(model) ? model : '' + $models.map((m) => m.id).includes(model) ? model : '' ); } @@ -45,7 +45,7 @@ {#if model.name === 'hr'}
{:else} - diff --git a/src/lib/components/chat/SettingsModal.svelte b/src/lib/components/chat/SettingsModal.svelte index 9d631f16..0e257af9 100644 --- a/src/lib/components/chat/SettingsModal.svelte +++ b/src/lib/components/chat/SettingsModal.svelte @@ -4,6 +4,7 @@ import { getOllamaModels } from '$lib/apis/ollama'; import { getOpenAIModels } from '$lib/apis/openai'; + import { getLiteLLMModels } from '$lib/apis/litellm'; import Modal from '../common/Modal.svelte'; import Account from './Settings/Account.svelte'; @@ -41,7 +42,15 @@ console.log(error); return null; }); + models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); + + const liteLLMModels = await getLiteLLMModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }); + + models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); } return models; diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 614fe959..11cf38f4 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -5,6 +5,8 @@ export const WEBUI_NAME = 'Open WebUI'; export const WEBUI_BASE_URL = dev ? `http://${location.hostname}:8080` : ``; export const WEBUI_API_BASE_URL = `${WEBUI_BASE_URL}/api/v1`; + +export const LITELLM_API_BASE_URL = `${WEBUI_BASE_URL}/litellm/api`; export const OLLAMA_API_BASE_URL = `${WEBUI_BASE_URL}/ollama/api`; export const OPENAI_API_BASE_URL = `${WEBUI_BASE_URL}/openai/api`; export const AUDIO_API_BASE_URL = `${WEBUI_BASE_URL}/audio/api/v1`; diff --git a/src/routes/(app)/+layout.svelte b/src/routes/(app)/+layout.svelte index af3bb31c..d34a09b7 100644 --- a/src/routes/(app)/+layout.svelte +++ b/src/routes/(app)/+layout.svelte @@ -12,6 +12,7 @@ import { getPrompts } from '$lib/apis/prompts'; import { getOpenAIModels } from '$lib/apis/openai'; + import { getLiteLLMModels } from '$lib/apis/litellm'; import { user, @@ -59,6 +60,13 @@ models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); + const liteLLMModels = await getLiteLLMModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }); + + models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); + return models; }; diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte index 9783ef5d..0bafd0d3 100644 --- a/src/routes/(app)/+page.svelte +++ b/src/routes/(app)/+page.svelte @@ -36,6 +36,7 @@ import ModelSelector from '$lib/components/chat/ModelSelector.svelte'; import Navbar from '$lib/components/layout/Navbar.svelte'; import { RAGTemplate } from '$lib/utils/rag'; + import { LITELLM_API_BASE_URL, OPENAI_API_BASE_URL } from '$lib/constants'; let stopResponseFlag = false; let autoScroll = true; @@ -277,9 +278,8 @@ } await Promise.all( - selectedModels.map(async (model) => { - console.log(model); - const modelTag = $models.filter((m) => m.name === model).at(0); + selectedModels.map(async (modelId) => { + const model = $models.filter((m) => m.id === modelId).at(0); // Create response message let responseMessageId = uuidv4(); @@ -289,7 +289,7 @@ childrenIds: [], role: 'assistant', content: '', - model: model, + model: model.id, timestamp: Math.floor(Date.now() / 1000) // Unix epoch }; @@ -305,12 +305,12 @@ ]; } - if (modelTag?.external) { + if (model?.external) { await sendPromptOpenAI(model, prompt, responseMessageId, _chatId); - } else if (modelTag) { + } else if (model) { await sendPromptOllama(model, prompt, responseMessageId, _chatId); } else { - toast.error(`Model ${model} not found`); + toast.error(`Model ${model.id} not found`); } }) ); @@ -319,6 +319,7 @@ }; const sendPromptOllama = async (model, userPrompt, responseMessageId, _chatId) => { + model = model.id; const responseMessage = history.messages[responseMessageId]; // Wait until history/message have been updated @@ -530,54 +531,58 @@ const responseMessage = history.messages[responseMessageId]; scrollToBottom(); - const res = await generateOpenAIChatCompletion(localStorage.token, { - model: model, - stream: true, - messages: [ - $settings.system - ? { - role: 'system', - content: $settings.system - } - : undefined, - ...messages - ] - .filter((message) => message) - .map((message, idx, arr) => ({ - role: message.role, - ...(message.files?.filter((file) => file.type === 'image').length > 0 ?? false + const res = await generateOpenAIChatCompletion( + localStorage.token, + { + model: model.id, + stream: true, + messages: [ + $settings.system ? { - content: [ - { - type: 'text', - text: - arr.length - 1 !== idx - ? message.content - : message?.raContent ?? message.content - }, - ...message.files - .filter((file) => file.type === 'image') - .map((file) => ({ - type: 'image_url', - image_url: { - url: file.url - } - })) - ] + role: 'system', + content: $settings.system } - : { - content: - arr.length - 1 !== idx ? message.content : message?.raContent ?? message.content - }) - })), - seed: $settings?.options?.seed ?? undefined, - stop: $settings?.options?.stop ?? undefined, - temperature: $settings?.options?.temperature ?? undefined, - top_p: $settings?.options?.top_p ?? undefined, - num_ctx: $settings?.options?.num_ctx ?? undefined, - frequency_penalty: $settings?.options?.repeat_penalty ?? undefined, - max_tokens: $settings?.options?.num_predict ?? undefined - }); + : undefined, + ...messages + ] + .filter((message) => message) + .map((message, idx, arr) => ({ + role: message.role, + ...(message.files?.filter((file) => file.type === 'image').length > 0 ?? false + ? { + content: [ + { + type: 'text', + text: + arr.length - 1 !== idx + ? message.content + : message?.raContent ?? message.content + }, + ...message.files + .filter((file) => file.type === 'image') + .map((file) => ({ + type: 'image_url', + image_url: { + url: file.url + } + })) + ] + } + : { + content: + arr.length - 1 !== idx ? message.content : message?.raContent ?? message.content + }) + })), + seed: $settings?.options?.seed ?? undefined, + stop: $settings?.options?.stop ?? undefined, + temperature: $settings?.options?.temperature ?? undefined, + top_p: $settings?.options?.top_p ?? undefined, + num_ctx: $settings?.options?.num_ctx ?? undefined, + frequency_penalty: $settings?.options?.repeat_penalty ?? undefined, + max_tokens: $settings?.options?.num_predict ?? undefined + }, + model.source === 'litellm' ? `${LITELLM_API_BASE_URL}/v1` : `${OPENAI_API_BASE_URL}` + ); if (res && res.ok) { const reader = res.body From af388dfe62ec2102e7c75afcf0878f93bdd2311b Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 23 Feb 2024 22:27:28 -0800 Subject: [PATCH 03/14] fix --- src/routes/(app)/+page.svelte | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte index dc098b88..31312e80 100644 --- a/src/routes/(app)/+page.svelte +++ b/src/routes/(app)/+page.svelte @@ -539,23 +539,12 @@ stream: true, messages: [ $settings.system - ? { - role: 'system', - content: $settings.system - } - : undefined, - ...messages.filter((message) => !message.deleted) - ] - .filter((message) => message) - .map((message, idx, arr) => ({ - role: message.role, - ...(message.files?.filter((file) => file.type === 'image').length > 0 ?? false ? { role: 'system', content: $settings.system } : undefined, - ...messages + ...messages.filter((message) => !message.deleted) ] .filter((message) => message) .map((message, idx, arr) => ({ From b5bd07a06a7f62e548c5dd18d23ba3c3eb1f19c2 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 23 Feb 2024 22:44:56 -0800 Subject: [PATCH 04/14] feat: secure litellm api --- backend/main.py | 21 +++++++++++++++++++-- backend/utils/utils.py | 11 +++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/backend/main.py b/backend/main.py index b9370a18..94145a97 100644 --- a/backend/main.py +++ b/backend/main.py @@ -4,9 +4,10 @@ import markdown import time -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, Depends from fastapi.staticfiles import StaticFiles from fastapi import HTTPException +from fastapi.responses import JSONResponse from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.middleware.cors import CORSMiddleware from starlette.exceptions import HTTPException as StarletteHTTPException @@ -19,10 +20,11 @@ from apps.openai.main import app as openai_app from apps.audio.main import app as audio_app from apps.images.main import app as images_app from apps.rag.main import app as rag_app - from apps.web.main import app as webui_app + from config import WEBUI_NAME, ENV, VERSION, CHANGELOG, FRONTEND_BUILD_DIR +from utils.utils import get_http_authorization_cred, get_current_user class SPAStaticFiles(StaticFiles): @@ -59,6 +61,21 @@ async def check_url(request: Request, call_next): return response +@litellm_app.middleware("http") +async def auth_middleware(request: Request, call_next): + auth_header = request.headers.get("Authorization", "") + + if ENV != "dev": + try: + user = get_current_user(get_http_authorization_cred(auth_header)) + print(user) + except Exception as e: + return JSONResponse(status_code=400, content={"detail": str(e)}) + + response = await call_next(request) + return response + + app.mount("/api/v1", webui_app) app.mount("/litellm/api", litellm_app) diff --git a/backend/utils/utils.py b/backend/utils/utils.py index c6d01814..7ae7e694 100644 --- a/backend/utils/utils.py +++ b/backend/utils/utils.py @@ -58,6 +58,17 @@ def extract_token_from_auth_header(auth_header: str): return auth_header[len("Bearer ") :] +def get_http_authorization_cred(auth_header: str): + try: + scheme, credentials = auth_header.split(" ") + return { + "scheme": scheme, + "credentials": credentials, + } + except: + raise ValueError(ERROR_MESSAGES.INVALID_TOKEN) + + def get_current_user( auth_token: HTTPAuthorizationCredentials = Depends(bearer_security), ): From 14dd0d11fb6d670c0e6d3c430541810c484dcdb1 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 00:21:53 -0800 Subject: [PATCH 05/14] feat: litellm yaml --- backend/.gitignore | 2 ++ backend/main.py | 25 +++++++++++++++++++++++-- test.json | 6 ++++++ 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 test.json diff --git a/backend/.gitignore b/backend/.gitignore index 16180123..1a2bc493 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -8,4 +8,6 @@ _test Pipfile data/* !data/config.json +!data/litellm/config.yaml + .webui_secret_key \ No newline at end of file diff --git a/backend/main.py b/backend/main.py index 94145a97..a432e9ed 100644 --- a/backend/main.py +++ b/backend/main.py @@ -2,7 +2,8 @@ from bs4 import BeautifulSoup import json import markdown import time - +import os +import sys from fastapi import FastAPI, Request, Depends from fastapi.staticfiles import StaticFiles @@ -13,6 +14,7 @@ from fastapi.middleware.cors import CORSMiddleware from starlette.exceptions import HTTPException as StarletteHTTPException +from litellm.proxy.proxy_server import ProxyConfig, initialize from litellm.proxy.proxy_server import app as litellm_app from apps.ollama.main import app as ollama_app @@ -38,6 +40,21 @@ class SPAStaticFiles(StaticFiles): raise ex +proxy_config = ProxyConfig() + + +async def config(): + router, model_list, general_settings = await proxy_config.load_config( + router=None, config_file_path="./data/litellm/config.yaml" + ) + + await initialize(config="./data/litellm/config.yaml", telemetry=False) + + +async def startup(): + await config() + + app = FastAPI(docs_url="/docs" if ENV == "dev" else None, redoc_url=None) origins = ["*"] @@ -51,6 +68,11 @@ app.add_middleware( ) +@app.on_event("startup") +async def on_startup(): + await startup() + + @app.middleware("http") async def check_url(request: Request, call_next): start_time = int(time.time()) @@ -79,7 +101,6 @@ async def auth_middleware(request: Request, call_next): app.mount("/api/v1", webui_app) app.mount("/litellm/api", litellm_app) - app.mount("/ollama/api", ollama_app) app.mount("/openai/api", openai_app) diff --git a/test.json b/test.json new file mode 100644 index 00000000..dccb6b26 --- /dev/null +++ b/test.json @@ -0,0 +1,6 @@ +{ + "model_name": "string", + "litellm_params": { + "model": "ollama/mistral" + } +} \ No newline at end of file From 43f9a827d38b039b65163c8517fc59fb9b55e226 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 00:29:03 -0800 Subject: [PATCH 06/14] fix: gitignore --- backend/.gitignore | 7 +++++-- backend/data/litellm/config.yaml | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 backend/data/litellm/config.yaml diff --git a/backend/.gitignore b/backend/.gitignore index 1a2bc493..ea83b34f 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -6,8 +6,11 @@ uploads *.db _test Pipfile -data/* -!data/config.json +!/data +/data/* +!/data/litellm +/data/litellm/* !data/litellm/config.yaml +!data/config.json .webui_secret_key \ No newline at end of file diff --git a/backend/data/litellm/config.yaml b/backend/data/litellm/config.yaml new file mode 100644 index 00000000..7d9d2b72 --- /dev/null +++ b/backend/data/litellm/config.yaml @@ -0,0 +1,4 @@ +general_settings: {} +litellm_settings: {} +model_list: [] +router_settings: {} From e8904ac3f7f63bbcdd7facd82f51338b26027fb8 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 01:03:28 -0800 Subject: [PATCH 07/14] feat: remove ollama dependency --- src/routes/(app)/+layout.svelte | 99 +++++++-------------------------- 1 file changed, 20 insertions(+), 79 deletions(-) diff --git a/src/routes/(app)/+layout.svelte b/src/routes/(app)/+layout.svelte index aab34a10..54922fd0 100644 --- a/src/routes/(app)/+layout.svelte +++ b/src/routes/(app)/+layout.svelte @@ -44,31 +44,28 @@ let showShortcuts = false; const getModels = async () => { - let models = []; - models.push( - ...(await getOllamaModels(localStorage.token).catch((error) => { - toast.error(error); - return []; - })) - ); + let models = await Promise.all([ + await getOllamaModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }), + await getOpenAIModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }), + await getLiteLLMModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }) + ]); - // $settings.OPENAI_API_BASE_URL ?? 'https://api.openai.com/v1', - // $settings.OPENAI_API_KEY - - const openAIModels = await getOpenAIModels(localStorage.token).catch((error) => { - console.log(error); - return null; - }); - - models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); - - const liteLLMModels = await getLiteLLMModels(localStorage.token).catch((error) => { - console.log(error); - return null; - }); - - models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); + models = models + .filter((models) => models) + .reduce((a, e, i, arr) => a.concat(e, ...(i < arr.length - 1 ? [{ name: 'hr' }] : [])), []); + // models.push(...(ollamaModels ? [{ name: 'hr' }, ...ollamaModels] : [])); + // models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); + // models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); return models; }; @@ -125,8 +122,6 @@ await models.set(await getModels()); }); - await setOllamaVersion(); - document.addEventListener('keydown', function (event) { const isCtrlPressed = event.ctrlKey || event.metaKey; // metaKey is for Cmd key on Mac // Check if the Shift key is pressed @@ -258,60 +253,6 @@ - {:else if checkVersion(REQUIRED_OLLAMA_VERSION, ollamaVersion ?? '0')} -
-
-
-
-
- Connection Issue or Update Needed -
- -
- Oops! It seems like your Ollama needs a little attention. We've detected either a connection hiccup or observed that you're using an older - version. Ensure you're on the latest Ollama version - (version - {REQUIRED_OLLAMA_VERSION} or higher) or check your connection. - -
- Trouble accessing Ollama? - - Click here for help. - -
-
- -
- - - -
-
-
-
-
{:else if localDBChats.length > 0}
Date: Sat, 24 Feb 2024 01:19:44 -0800 Subject: [PATCH 08/14] fix: pip dependency --- backend/requirements.txt | 2 +- .../chat/Settings/Connections.svelte | 11 +++--- src/lib/components/chat/SettingsModal.svelte | 36 +++++++++---------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 75d368d8..7ec45b04 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -16,7 +16,7 @@ aiohttp peewee bcrypt -litellm[proxy] +litellm langchain langchain-community diff --git a/src/lib/components/chat/Settings/Connections.svelte b/src/lib/components/chat/Settings/Connections.svelte index 462d3005..486fd9b7 100644 --- a/src/lib/components/chat/Settings/Connections.svelte +++ b/src/lib/components/chat/Settings/Connections.svelte @@ -3,7 +3,7 @@ import { createEventDispatcher, onMount } from 'svelte'; const dispatch = createEventDispatcher(); - import { getOllamaAPIUrl, updateOllamaAPIUrl } from '$lib/apis/ollama'; + import { getOllamaAPIUrl, getOllamaVersion, updateOllamaAPIUrl } from '$lib/apis/ollama'; import { getOpenAIKey, getOpenAIUrl, updateOpenAIKey, updateOpenAIUrl } from '$lib/apis/openai'; import toast from 'svelte-french-toast'; @@ -24,11 +24,14 @@ const updateOllamaAPIUrlHandler = async () => { API_BASE_URL = await updateOllamaAPIUrl(localStorage.token, API_BASE_URL); - const _models = await getModels('ollama'); - if (_models.length > 0) { + const ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => { + return null; + }); + + if (ollamaVersion) { toast.success('Server connection verified'); - await models.set(_models); + await models.set(await getModels()); } }; diff --git a/src/lib/components/chat/SettingsModal.svelte b/src/lib/components/chat/SettingsModal.svelte index 0e257af9..b112180b 100644 --- a/src/lib/components/chat/SettingsModal.svelte +++ b/src/lib/components/chat/SettingsModal.svelte @@ -28,31 +28,29 @@ let selectedTab = 'general'; - const getModels = async (type = 'all') => { - const models = []; - models.push( - ...(await getOllamaModels(localStorage.token).catch((error) => { - toast.error(error); - return []; - })) - ); - - if (type === 'all') { - const openAIModels = await getOpenAIModels(localStorage.token).catch((error) => { + const getModels = async () => { + let models = await Promise.all([ + await getOllamaModels(localStorage.token).catch((error) => { console.log(error); return null; - }); - - models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); - - const liteLLMModels = await getLiteLLMModels(localStorage.token).catch((error) => { + }), + await getOpenAIModels(localStorage.token).catch((error) => { console.log(error); return null; - }); + }), + await getLiteLLMModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }) + ]); - models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); - } + models = models + .filter((models) => models) + .reduce((a, e, i, arr) => a.concat(e, ...(i < arr.length - 1 ? [{ name: 'hr' }] : [])), []); + // models.push(...(ollamaModels ? [{ name: 'hr' }, ...ollamaModels] : [])); + // models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); + // models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); return models; }; From a74d47b6fc7a9b7fbd3641c08a516ed7c7d31cf5 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 01:27:44 -0800 Subject: [PATCH 09/14] Update requirements.txt --- backend/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/requirements.txt b/backend/requirements.txt index 7ec45b04..341369fc 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -17,6 +17,7 @@ peewee bcrypt litellm +apscheduler langchain langchain-community From fdb6499bfa697c95095b53dba259f4f760875c74 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 01:49:19 -0800 Subject: [PATCH 10/14] fix: error handling --- src/routes/(app)/+page.svelte | 62 ++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte index 31312e80..79fbf3cd 100644 --- a/src/routes/(app)/+page.svelte +++ b/src/routes/(app)/+page.svelte @@ -133,6 +133,10 @@ selectedModels = ['']; } + selectedModels = selectedModels.map((modelId) => + $models.map((m) => m.id).includes(modelId) ? modelId : '' + ); + let _settings = JSON.parse(localStorage.getItem('settings') ?? '{}'); settings.set({ ..._settings @@ -151,6 +155,10 @@ const submitPrompt = async (userPrompt, _user = null) => { console.log('submitPrompt', $chatId); + selectedModels = selectedModels.map((modelId) => + $models.map((m) => m.id).includes(modelId) ? modelId : '' + ); + if (selectedModels.includes('')) { toast.error('Model not selected'); } else if (messages.length != 0 && messages.at(-1).done != true) { @@ -282,36 +290,38 @@ selectedModels.map(async (modelId) => { const model = $models.filter((m) => m.id === modelId).at(0); - // Create response message - let responseMessageId = uuidv4(); - let responseMessage = { - parentId: parentId, - id: responseMessageId, - childrenIds: [], - role: 'assistant', - content: '', - model: model.id, - timestamp: Math.floor(Date.now() / 1000) // Unix epoch - }; + if (model) { + // Create response message + let responseMessageId = uuidv4(); + let responseMessage = { + parentId: parentId, + id: responseMessageId, + childrenIds: [], + role: 'assistant', + content: '', + model: model.id, + timestamp: Math.floor(Date.now() / 1000) // Unix epoch + }; - // Add message to history and Set currentId to messageId - history.messages[responseMessageId] = responseMessage; - history.currentId = responseMessageId; + // Add message to history and Set currentId to messageId + history.messages[responseMessageId] = responseMessage; + history.currentId = responseMessageId; - // Append messageId to childrenIds of parent message - if (parentId !== null) { - history.messages[parentId].childrenIds = [ - ...history.messages[parentId].childrenIds, - responseMessageId - ]; - } + // Append messageId to childrenIds of parent message + if (parentId !== null) { + history.messages[parentId].childrenIds = [ + ...history.messages[parentId].childrenIds, + responseMessageId + ]; + } - if (model?.external) { - await sendPromptOpenAI(model, prompt, responseMessageId, _chatId); - } else if (model) { - await sendPromptOllama(model, prompt, responseMessageId, _chatId); + if (model?.external) { + await sendPromptOpenAI(model, prompt, responseMessageId, _chatId); + } else if (model) { + await sendPromptOllama(model, prompt, responseMessageId, _chatId); + } } else { - toast.error(`Model ${model.id} not found`); + toast.error(`Model ${modelId} not found`); } }) ); From 32da2aa511aa895b088e6343e6fb6bd6f2fdd9ce Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 13:11:56 -0800 Subject: [PATCH 11/14] Update config.py --- backend/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/config.py b/backend/config.py index fadae68c..b16913c1 100644 --- a/backend/config.py +++ b/backend/config.py @@ -83,8 +83,6 @@ for version in soup.find_all("h2"): # Find the next sibling that is a h3 tag (section title) current = version.find_next_sibling() - print(current) - while current and current.name != "h2": if current.name == "h3": section_title = current.get_text().lower() # e.g., "added", "fixed" From e15e015db162f7236a7d38bbc7992a8076ac568a Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 15:48:14 -0800 Subject: [PATCH 12/14] fix: images --- backend/apps/images/main.py | 8 ++++---- src/lib/components/chat/Settings/Images.svelte | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/apps/images/main.py b/backend/apps/images/main.py index 39d3f96a..3ea8b171 100644 --- a/backend/apps/images/main.py +++ b/backend/apps/images/main.py @@ -49,7 +49,7 @@ async def toggle_enabled(request: Request, user=Depends(get_admin_user)): app.state.ENABLED = not app.state.ENABLED return app.state.ENABLED except Exception as e: - raise HTTPException(status_code=r.status_code, detail=ERROR_MESSAGES.DEFAULT(e)) + raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) class UrlUpdateForm(BaseModel): @@ -109,7 +109,7 @@ def get_models(user=Depends(get_current_user)): models = r.json() return models except Exception as e: - raise HTTPException(status_code=r.status_code, detail=ERROR_MESSAGES.DEFAULT(e)) + raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) @app.get("/models/default") @@ -120,7 +120,7 @@ async def get_default_model(user=Depends(get_admin_user)): return {"model": options["sd_model_checkpoint"]} except Exception as e: - raise HTTPException(status_code=r.status_code, detail=ERROR_MESSAGES.DEFAULT(e)) + raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) class UpdateModelForm(BaseModel): @@ -190,4 +190,4 @@ def generate_image( return r.json() except Exception as e: print(e) - raise HTTPException(status_code=r.status_code, detail=ERROR_MESSAGES.DEFAULT(e)) + raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) diff --git a/src/lib/components/chat/Settings/Images.svelte b/src/lib/components/chat/Settings/Images.svelte index 94cb3213..cc827454 100644 --- a/src/lib/components/chat/Settings/Images.svelte +++ b/src/lib/components/chat/Settings/Images.svelte @@ -32,9 +32,11 @@ const getModels = async () => { models = await getDiffusionModels(localStorage.token).catch((error) => { toast.error(error); - return null; + return []; + }); + selectedModel = await getDefaultDiffusionModel(localStorage.token).catch((error) => { + return ''; }); - selectedModel = await getDefaultDiffusionModel(localStorage.token); }; const updateAUTOMATIC1111UrlHandler = async () => { From a453be07e11aada1821f21d8e75b9410992492d8 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 17:22:28 -0800 Subject: [PATCH 13/14] Update CHANGELOG.md Co-Authored-By: Alex Gartner <2747955+gartnera@users.noreply.github.com> --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48a60634..2a35d723 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.103] - UNRELEASED + +### Added + +- **Built-in LiteLLM Proxy**: Open WebUI now ships with LiteLLM Proxy. +- **Image Generation Enhancements**: Advanced Settings + Image Preview Feature. + +### Fixed + +- Issue with RAG scan that stops loading documents as soon as it reaches a file with unsupported mime type (or any other exceptions). (#866) + +### Changed + +- Ollama is no longer required to run Open WebUI. +- Our documentation can be found here https://docs.openwebui.com/ + ## [0.1.102] - 2024-02-22 ### Added From ec6f53e29837440da799afe3b51e9d03ef56262a Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 17:56:53 -0800 Subject: [PATCH 14/14] feat: litellm proxy frontend integration --- backend/apps/images/main.py | 2 + package.json | 2 +- src/lib/apis/litellm/index.ts | 108 ++- src/lib/components/chat/Settings/About.svelte | 16 +- .../chat/Settings/Connections.svelte | 164 ++-- .../components/chat/Settings/Models.svelte | 891 ++++++++++++------ 6 files changed, 823 insertions(+), 360 deletions(-) diff --git a/backend/apps/images/main.py b/backend/apps/images/main.py index 3ea8b171..9322ea7f 100644 --- a/backend/apps/images/main.py +++ b/backend/apps/images/main.py @@ -109,6 +109,7 @@ def get_models(user=Depends(get_current_user)): models = r.json() return models except Exception as e: + app.state.ENABLED = False raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) @@ -120,6 +121,7 @@ async def get_default_model(user=Depends(get_admin_user)): return {"model": options["sd_model_checkpoint"]} except Exception as e: + app.state.ENABLED = False raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) diff --git a/package.json b/package.json index 30549fdd..7938558d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-webui", - "version": "0.1.102", + "version": "0.1.103", "private": true, "scripts": { "dev": "vite dev --host", diff --git a/src/lib/apis/litellm/index.ts b/src/lib/apis/litellm/index.ts index 87d2654e..6466ee35 100644 --- a/src/lib/apis/litellm/index.ts +++ b/src/lib/apis/litellm/index.ts @@ -17,7 +17,7 @@ export const getLiteLLMModels = async (token: string = '') => { }) .catch((err) => { console.log(err); - error = `OpenAI: ${err?.error?.message ?? 'Network Problem'}`; + error = `LiteLLM: ${err?.error?.message ?? 'Network Problem'}`; return []; }); @@ -40,3 +40,109 @@ export const getLiteLLMModels = async (token: string = '') => { }) : models; }; + +export const getLiteLLMModelInfo = async (token: string = '') => { + let error = null; + + const res = await fetch(`${LITELLM_API_BASE_URL}/model/info`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = `LiteLLM: ${err?.error?.message ?? 'Network Problem'}`; + return []; + }); + + if (error) { + throw error; + } + + const models = Array.isArray(res) ? res : res?.data ?? null; + + return models; +}; + +type AddLiteLLMModelForm = { + name: string; + model: string; + api_base: string; + api_key: string; + rpm: string; +}; + +export const addLiteLLMModel = async (token: string = '', payload: AddLiteLLMModelForm) => { + let error = null; + + const res = await fetch(`${LITELLM_API_BASE_URL}/model/new`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + }, + body: JSON.stringify({ + model_name: payload.name, + litellm_params: { + model: payload.model, + ...(payload.api_base === '' ? {} : { api_base: payload.api_base }), + ...(payload.api_key === '' ? {} : { api_key: payload.api_key }), + ...(isNaN(parseInt(payload.rpm)) ? {} : { rpm: parseInt(payload.rpm) }) + } + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = `LiteLLM: ${err?.error?.message ?? 'Network Problem'}`; + return []; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const deleteLiteLLMModel = async (token: string = '', id: string) => { + let error = null; + + const res = await fetch(`${LITELLM_API_BASE_URL}/model/delete`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + }, + body: JSON.stringify({ + id: id + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = `LiteLLM: ${err?.error?.message ?? 'Network Problem'}`; + return []; + }); + + if (error) { + throw error; + } + + return res; +}; diff --git a/src/lib/components/chat/Settings/About.svelte b/src/lib/components/chat/Settings/About.svelte index 6933ed3b..60c0d716 100644 --- a/src/lib/components/chat/Settings/About.svelte +++ b/src/lib/components/chat/Settings/About.svelte @@ -38,16 +38,18 @@
-
+ {#if ollamaVersion} +
-
-
Ollama Version
-
-
- {ollamaVersion ?? 'N/A'} +
+
Ollama Version
+
+
+ {ollamaVersion ?? 'N/A'} +
-
+ {/if}
diff --git a/src/lib/components/chat/Settings/Connections.svelte b/src/lib/components/chat/Settings/Connections.svelte index 486fd9b7..db24c8a0 100644 --- a/src/lib/components/chat/Settings/Connections.svelte +++ b/src/lib/components/chat/Settings/Connections.svelte @@ -15,6 +15,9 @@ let OPENAI_API_KEY = ''; let OPENAI_API_BASE_URL = ''; + let showOpenAI = false; + let showLiteLLM = false; + const updateOpenAIHandler = async () => { OPENAI_API_BASE_URL = await updateOpenAIUrl(localStorage.token, OPENAI_API_BASE_URL); OPENAI_API_KEY = await updateOpenAIKey(localStorage.token, OPENAI_API_KEY); @@ -45,7 +48,7 @@
{ updateOpenAIHandler(); dispatch('save'); @@ -56,81 +59,100 @@ // }); }} > -
-
Ollama API URL
-
-
- +
+
+
+
+
OpenAI API
+ +
+ + {#if showOpenAI} +
+
API Key
+
+
+ +
+
+
+ +
+
API Base URL
+
+
+ +
+
+
+ WebUI will make requests to '{OPENAI_API_BASE_URL}/chat' +
+
+ {/if}
- -
- -
- Trouble accessing Ollama? - - Click here for help. - -
-
- -
- -
-
-
OpenAI API Key
-
-
- -
+ + + +
-
-
-
OpenAI API Base URL
-
-
- -
-
- WebUI will make requests to '{OPENAI_API_BASE_URL}/chat' + Trouble accessing Ollama? + + Click here for help. +
diff --git a/src/lib/components/chat/Settings/Models.svelte b/src/lib/components/chat/Settings/Models.svelte index c7ca7908..455927df 100644 --- a/src/lib/components/chat/Settings/Models.svelte +++ b/src/lib/components/chat/Settings/Models.svelte @@ -2,14 +2,33 @@ import queue from 'async/queue'; import toast from 'svelte-french-toast'; - import { createModel, deleteModel, pullModel } from '$lib/apis/ollama'; + import { createModel, deleteModel, getOllamaVersion, pullModel } from '$lib/apis/ollama'; import { WEBUI_API_BASE_URL, WEBUI_BASE_URL } from '$lib/constants'; import { WEBUI_NAME, models, user } from '$lib/stores'; import { splitStream } from '$lib/utils'; + import { onMount } from 'svelte'; + import { addLiteLLMModel, deleteLiteLLMModel, getLiteLLMModelInfo } from '$lib/apis/litellm'; export let getModels: Function; + let showLiteLLM = false; + let showLiteLLMParams = false; + + let liteLLMModelInfo = []; + + let liteLLMModel = ''; + let liteLLMModelName = ''; + let liteLLMAPIBase = ''; + let liteLLMAPIKey = ''; + let liteLLMRPM = ''; + + let deleteLiteLLMModelId = ''; + + $: liteLLMModelName = liteLLMModel; + // Models + let showExperimentalOllama = false; + let ollamaVersion = ''; const MAX_PARALLEL_DOWNLOADS = 3; const modelDownloadQueue = queue( (task: { modelName: string }, cb) => @@ -286,256 +305,184 @@ opts.callback({ success: true, modelName: opts.modelName }); } }; + + const addLiteLLMModelHandler = async () => { + if (!liteLLMModelInfo.find((info) => info.model_name === liteLLMModelName)) { + const res = await addLiteLLMModel(localStorage.token, { + name: liteLLMModelName, + model: liteLLMModel, + api_base: liteLLMAPIBase, + api_key: liteLLMAPIKey, + rpm: liteLLMRPM + }).catch((error) => { + toast.error(error); + return null; + }); + + if (res) { + if (res.message) { + toast.success(res.message); + } + } + } else { + toast.error(`Model ${liteLLMModelName} already exists.`); + } + + liteLLMModelName = ''; + liteLLMModel = ''; + liteLLMAPIBase = ''; + liteLLMAPIKey = ''; + liteLLMRPM = ''; + + liteLLMModelInfo = await getLiteLLMModelInfo(localStorage.token); + models.set(await getModels()); + }; + + const deleteLiteLLMModelHandler = async () => { + const res = await deleteLiteLLMModel(localStorage.token, deleteLiteLLMModelId).catch( + (error) => { + toast.error(error); + return null; + } + ); + + if (res) { + if (res.message) { + toast.success(res.message); + } + } + + deleteLiteLLMModelId = ''; + liteLLMModelInfo = await getLiteLLMModelInfo(localStorage.token); + models.set(await getModels()); + }; + + onMount(async () => { + ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => false); + liteLLMModelInfo = await getLiteLLMModelInfo(localStorage.token); + });
-
-
-
Pull a model from Ollama.com
-
-
- -
- -
- -
- To access the available model names for downloading, click here. -
- - {#if Object.keys(modelDownloadStatus).length > 0} - {#each Object.keys(modelDownloadStatus) as model} -
-
{model}
-
-
- {modelDownloadStatus[model].pullProgress ?? 0}% -
-
- {modelDownloadStatus[model].digest} -
-
-
- {/each} - {/if} -
- -
- -
-
Delete a model
-
-
- -
- -
-
- -
- - { - uploadModelHandler(); - }} - > -
-
- Upload a GGUF model (Experimental) -
- - -
- -
-
- {#if modelUploadMode === 'file'} -
- { - console.log(modelInputFile); - }} - accept=".gguf" - required - hidden - /> - - -
- {:else} -
- -
- {/if} -
- - {#if (modelUploadMode === 'file' && modelInputFile && modelInputFile.length > 0) || (modelUploadMode === 'url' && modelFileUrl !== '')} - +
+ +
+ To access the available model names for downloading, click here. +
+ + {#if Object.keys(modelDownloadStatus).length > 0} + {#each Object.keys(modelDownloadStatus) as model} +
+
{model}
+
+
+ {modelDownloadStatus[model].pullProgress ?? 0}% +
+
+ {modelDownloadStatus[model].digest} +
+
- {:else} + {/each} + {/if} +
+ +
+
Delete a model
+
+
+ +
+ +
+
+ +
+
+
Experimental
+ +
+
+ + {#if showExperimentalOllama} + { + uploadModelHandler(); + }} + > +
+
Upload a GGUF model
+ + +
+ +
+
+ {#if modelUploadMode === 'file'} +
+ { + console.log(modelInputFile); + }} + accept=".gguf" + required + hidden + /> + + +
+ {:else} +
+ +
+ {/if} +
+ + {#if (modelUploadMode === 'file' && modelInputFile && modelInputFile.length > 0) || (modelUploadMode === 'url' && modelFileUrl !== '')} + + {/if} +
+ + {#if (modelUploadMode === 'file' && modelInputFile && modelInputFile.length > 0) || (modelUploadMode === 'url' && modelFileUrl !== '')} +
+
+
Modelfile Content
+