tdpeuter/open-webui
tdpeuter/open-webui
1
Fork 0
forked from open-webui/open-webui
Code Issues Pull requests 1 Releases 2 Packages Activity Actions

Merge pull request #212 from ollama-webui/multimodal

Browse source 
feat: multimodal support
This commit is contained in:
Timothy Jaeryang Baek 2023-12-13 20:08:50 -05:00 committed by GitHub
commit 346b0df811
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 131 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
README.md
View file

@ -35,6 +35,8 @@ Also check our sibling project, [OllamaHub](https://ollamahub.com/), where you c
- 🤖 **Multiple Model Support**: Seamlessly switch between different chat models for diverse interactions.
- 🔄 **Multi-Modal Support**: Seamlessly engage with models that support multimodal interactions, including images (e.g., LLava).
- 🧩 **Modelfile Builder**: Easily create Ollama modelfiles via the web UI. Create and add characters/agents, customize chat elements, and import modelfiles effortlessly through [OllamaHub](https://ollamahub.com/) integration.
- ⚙️ **Many Models Conversations**: Effortlessly engage with various models simultaneously, harnessing their unique strengths for optimal responses. Enhance your experience by leveraging a diverse set of models in parallel.
@ -234,7 +236,6 @@ See [TROUBLESHOOTING.md](/TROUBLESHOOTING.md) for information on how to troubles
Here are some exciting tasks on our roadmap:
- 🔄 **Multi-Modal Support**: Seamlessly engage with models that support multimodal interactions, including images (e.g., LLava).
- 📚 **RAG Integration**: Experience first-class retrieval augmented generation support, enabling chat with your documents.
- 🔐 **Access Control**: Securely manage requests to Ollama by utilizing the backend as a reverse proxy gateway, ensuring only authenticated users can send specific requests.
- 🧪 **Research-Centric Features**: Empower researchers in the fields of LLM and HCI with a comprehensive web UI for conducting user studies. Stay tuned for ongoing feature enhancements (e.g., surveys, analytics, and participant tracking) to facilitate their research.

BIN
demo.gif
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.5 MiB

After

Width:  |  Height:  |  Size: 6.3 MiB

13
src/lib/components/chat/MessageInput.svelte
View file

@ -14,7 +14,7 @@
export let files = [];
export let fileUploadEnabled = false;
export let fileUploadEnabled = true;
export let speechRecognitionEnabled = true;
export let speechRecognitionListening = false;
@ -84,11 +84,12 @@
};
</script>
<div class="fixed bottom-0 w-full bg-white dark:bg-gray-800">
<div class=" absolute right-0 left-0 bottom-0 mb-20">
<div class="max-w-3xl px-2.5 pt-2.5 -mb-0.5 mx-auto inset-x-0">
<div class="fixed bottom-0 w-full">
<div class="px-2.5 pt-2.5 -mb-0.5 mx-auto inset-x-0 bg-transparent flex justify-center">
{#if messages.length == 0 && suggestionPrompts.length !== 0}
<div class="max-w-3xl">
<Suggestions {suggestionPrompts} {submitPrompt} />
</div>
{/if}
{#if autoScroll === false && messages.length > 0}
@ -116,8 +117,7 @@
</div>
{/if}
</div>
</div>
<div>
<div class="bg-white dark:bg-gray-800">
<div class="max-w-3xl px-2.5 -mb-0.5 mx-auto inset-x-0">
<div class="bg-gradient-to-t from-white dark:from-gray-800 from-40% pb-2">
<input
@ -136,6 +136,7 @@
}
];
inputFiles = null;
filesInputElement.value = '';
};
if (

38
src/lib/components/chat/Messages.svelte
View file

@ -15,6 +15,7 @@
export let sendPrompt: Function;
export let regenerateResponse: Function;
export let bottomPadding = false;
export let autoScroll;
export let selectedModels;
export let history = {};
@ -31,6 +32,13 @@
})();
}
$: if (autoScroll && bottomPadding) {
(async () => {
await tick();
window.scrollTo({ top: document.body.scrollHeight, behavior: 'smooth' });
})();
}
const speakMessage = (message) => {
const speak = new SpeechSynthesisUtterance(message);
speechSynthesis.speak(speak);
@ -184,7 +192,8 @@
parentId: history.messages[messageId].parentId,
childrenIds: [],
role: 'user',
content: userPrompt
content: userPrompt,
...(history.messages[messageId].files && { files: history.messages[messageId].files })
};
let messageParentId = history.messages[messageId].parentId;
@ -425,6 +434,18 @@
class="prose chat-{message.role} w-full max-w-full dark:prose-invert prose-headings:my-0 prose-p:my-0 prose-p:-mb-4 prose-pre:my-0 prose-table:my-0 prose-blockquote:my-0 prose-img:my-0 prose-ul:-my-4 prose-ol:-my-4 prose-li:-my-3 prose-ul:-mb-6 prose-ol:-mb-6 prose-li:-mb-4 whitespace-pre-line"
>
{#if message.role == 'user'}
{#if message.files}
<div class="my-3 w-full flex overflow-x-auto space-x-2">
{#each message.files as file}
<div>
{#if file.type === 'image'}
<img src={file.url} alt="input" class=" max-h-96 rounded-lg" />
{/if}
</div>
{/each}
</div>
{/if}
{#if message?.edit === true}
<div class=" w-full">
<textarea
@ -458,17 +479,6 @@
</div>
{:else}
<div class="w-full">
{#if message.files}
<div class="my-3">
{#each message.files as file}
<div>
{#if file.type === 'image'}
<img src={file.url} alt="input" class=" max-h-96" />
{/if}
</div>
{/each}
</div>
{/if}
<pre id="user-message">{message.content}</pre>
<div class=" flex justify-start space-x-1">
@ -889,4 +899,8 @@
</div>
</div>
{/each}
{#if bottomPadding}
<div class=" mb-10" />
{/if}
{/if}

70
src/routes/(app)/+page.svelte
View file

@ -50,6 +50,10 @@
messages = [];
}
$: if (files) {
console.log(files);
}
onMount(async () => {
await chatId.set(uuidv4());
@ -106,7 +110,6 @@
const sendPromptOllama = async (model, userPrompt, parentId, _chatId) => {
console.log('sendPromptOllama');
let responseMessageId = uuidv4();
let responseMessage = {
parentId: parentId,
id: responseMessageId,
@ -126,38 +129,8 @@
}
await tick();
window.scrollTo({ top: document.body.scrollHeight });
// const res = await fetch(`${$settings?.API_BASE_URL ?? OLLAMA_API_BASE_URL}/generate`, {
// method: 'POST',
// headers: {
// 'Content-Type': 'text/event-stream',
// ...($settings.authHeader && { Authorization: $settings.authHeader }),
// ...($user && { Authorization: `Bearer ${localStorage.token}` })
// },
// body: JSON.stringify({
// model: model,
// prompt: userPrompt,
// system: $settings.system ?? undefined,
// options: {
// seed: $settings.seed ?? undefined,
// temperature: $settings.temperature ?? undefined,
// repeat_penalty: $settings.repeat_penalty ?? undefined,
// top_k: $settings.top_k ?? undefined,
// top_p: $settings.top_p ?? undefined,
// num_ctx: $settings.num_ctx ?? undefined,
// ...($settings.options ?? {})
// },
// format: $settings.requestFormat ?? undefined,
// context:
// history.messages[parentId] !== null &&
// history.messages[parentId].parentId in history.messages
// ? history.messages[history.messages[parentId].parentId]?.context ?? undefined
// : undefined
// })
// });
const res = await fetch(`${$settings?.API_BASE_URL ?? OLLAMA_API_BASE_URL}/chat`, {
method: 'POST',
headers: {
@ -177,7 +150,15 @@
...messages
]
.filter((message) => message)
.map((message) => ({ role: message.role, content: message.content })),
.map((message) => ({
role: message.role,
content: message.content,
...(message.files && {
images: message.files
.filter((file) => file.type === 'image')
.map((file) => file.url.slice(file.url.indexOf(',') + 1))
})
})),
options: {
seed: $settings.seed ?? undefined,
temperature: $settings.temperature ?? undefined,
@ -350,7 +331,27 @@
...messages
]
.filter((message) => message)
.map((message) => ({ role: message.role, content: message.content })),
.map((message) => ({
role: message.role,
...(message.files
? {
content: [
{
type: 'text',
text: message.content
},
...message.files
.filter((file) => file.type === 'image')
.map((file) => ({
type: 'image_url',
image_url: {
url: file.url
}
}))
]
}
: { content: message.content })
})),
temperature: $settings.temperature ?? undefined,
top_p: $settings.top_p ?? undefined,
num_ctx: $settings.num_ctx ?? undefined,
@ -579,6 +580,7 @@
bind:history
bind:messages
bind:autoScroll
bottomPadding={files.length > 0}
{sendPrompt}
{regenerateResponse}
/>
@ -586,8 +588,8 @@
</div>
<MessageInput
bind:prompt
bind:files
bind:prompt
bind:autoScroll
suggestionPrompts={selectedModelfile?.suggestionPrompts ?? [
{

81
src/routes/(app)/c/[id]/+page.svelte
View file

@ -51,17 +51,6 @@
messages = [];
}
// onMount(async () => {
// let chat = await loadChat();
// await tick();
// if (chat) {
// loaded = true;
// } else {
// await goto('/');
// }
// });
$: if ($page.params.id) {
(async () => {
let chat = await loadChat();
@ -133,7 +122,6 @@
const sendPromptOllama = async (model, userPrompt, parentId, _chatId) => {
console.log('sendPromptOllama');
let responseMessageId = uuidv4();
let responseMessage = {
parentId: parentId,
id: responseMessageId,
@ -153,38 +141,8 @@
}
await tick();
window.scrollTo({ top: document.body.scrollHeight });
// const res = await fetch(`${$settings?.API_BASE_URL ?? OLLAMA_API_BASE_URL}/generate`, {
// method: 'POST',
// headers: {
// 'Content-Type': 'text/event-stream',
// ...($settings.authHeader && { Authorization: $settings.authHeader }),
// ...($user && { Authorization: `Bearer ${localStorage.token}` })
// },
// body: JSON.stringify({
// model: model,
// prompt: userPrompt,
// system: $settings.system ?? undefined,
// options: {
// seed: $settings.seed ?? undefined,
// temperature: $settings.temperature ?? undefined,
// repeat_penalty: $settings.repeat_penalty ?? undefined,
// top_k: $settings.top_k ?? undefined,
// top_p: $settings.top_p ?? undefined,
// num_ctx: $settings.num_ctx ?? undefined,
// ...($settings.options ?? {})
// },
// format: $settings.requestFormat ?? undefined,
// context:
// history.messages[parentId] !== null &&
// history.messages[parentId].parentId in history.messages
// ? history.messages[history.messages[parentId].parentId]?.context ?? undefined
// : undefined
// })
// });
const res = await fetch(`${$settings?.API_BASE_URL ?? OLLAMA_API_BASE_URL}/chat`, {
method: 'POST',
headers: {
@ -204,7 +162,15 @@
...messages
]
.filter((message) => message)
.map((message) => ({ role: message.role, content: message.content })),
.map((message) => ({
role: message.role,
content: message.content,
...(message.files && {
images: message.files
.filter((file) => file.type === 'image')
.map((file) => file.url.slice(file.url.indexOf(',') + 1))
})
})),
options: {
seed: $settings.seed ?? undefined,
temperature: $settings.temperature ?? undefined,
@ -377,7 +343,27 @@
...messages
]
.filter((message) => message)
.map((message) => ({ role: message.role, content: message.content })),
.map((message) => ({
role: message.role,
...(message.files
? {
content: [
{
type: 'text',
text: message.content
},
...message.files
.filter((file) => file.type === 'image')
.map((file) => ({
type: 'image_url',
image_url: {
url: file.url
}
}))
]
}
: { content: message.content })
})),
temperature: $settings.temperature ?? undefined,
top_p: $settings.top_p ?? undefined,
num_ctx: $settings.num_ctx ?? undefined,
@ -392,12 +378,9 @@
while (true) {
const { value, done } = await reader.read();
if (done || stopResponseFlag) {
if (stopResponseFlag) {
if (done || stopResponseFlag || _chatId !== $chatId) {
responseMessage.done = true;
messages = messages;
}
break;
}
@ -610,6 +593,7 @@
bind:history
bind:messages
bind:autoScroll
bottomPadding={files.length > 0}
{sendPrompt}
{regenerateResponse}
/>
@ -617,6 +601,7 @@
</div>
<MessageInput
bind:files
bind:prompt
bind:autoScroll
suggestionPrompts={selectedModelfile?.suggestionPrompts ?? [