feat: # to import doc

This commit is contained in:
Timothy J. Baek 2024-01-07 23:43:32 -08:00
parent 2603ac30bc
commit cc3f84f916
11 changed files with 894 additions and 32 deletions

View file

@ -0,0 +1,177 @@
import { WEBUI_API_BASE_URL } from '$lib/constants';
export const createNewDoc = async (
token: string,
collection_name: string,
filename: string,
name: string,
title: string
) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/documents/create`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify({
collection_name: collection_name,
filename: filename,
name: name,
title: title
})
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const getDocs = async (token: string = '') => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/documents/`, {
method: 'GET',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.then((json) => {
return json;
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const getDocByName = async (token: string, name: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/documents/name/${name}`, {
method: 'GET',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.then((json) => {
return json;
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};
type DocUpdateForm = {
name: string;
title: string;
};
export const updateDocByName = async (token: string, name: string, form: DocUpdateForm) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/prompts/name/${name}/update`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify({
name: form.name,
title: form.title
})
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.then((json) => {
return json;
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const deleteDocByName = async (token: string, name: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/documents/name/${name}/delete`, {
method: 'DELETE',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.then((json) => {
return json;
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};

View file

@ -0,0 +1,6 @@
<div class=" text-center text-6xl mb-3">📄</div>
<div class="text-center dark:text-white text-2xl font-semibold z-50">Add Files</div>
<div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
Drop any files here to add to the conversation
</div>

View file

@ -7,6 +7,9 @@
import Prompts from './MessageInput/PromptCommands.svelte';
import Suggestions from './MessageInput/Suggestions.svelte';
import { uploadDocToVectorDB } from '$lib/apis/rag';
import AddFilesPlaceholder from '../AddFilesPlaceholder.svelte';
import { SUPPORTED_FILE_TYPE } from '$lib/constants';
import Documents from './MessageInput/Documents.svelte';
export let submitPrompt: Function;
export let stopResponse: Function;
@ -16,6 +19,7 @@
let filesInputElement;
let promptsElement;
let documentsElement;
let inputFiles;
let dragged = false;
@ -143,14 +147,7 @@
const file = inputFiles[0];
if (['image/gif', 'image/jpeg', 'image/png'].includes(file['type'])) {
reader.readAsDataURL(file);
} else if (
[
'application/pdf',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'text/plain',
'text/csv'
].includes(file['type'])
) {
} else if (SUPPORTED_FILE_TYPE.includes(file['type'])) {
uploadDoc(file);
} else {
toast.error(`Unsupported File Type '${file['type']}'.`);
@ -179,12 +176,7 @@
<div class="absolute rounded-xl w-full h-full backdrop-blur bg-gray-800/40 flex justify-center">
<div class="m-auto pt-64 flex flex-col justify-center">
<div class="max-w-md">
<div class=" text-center text-6xl mb-3">🗂️</div>
<div class="text-center dark:text-white text-2xl font-semibold z-50">Add Files</div>
<div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
Drop any files/images here to add to the conversation
</div>
<AddFilesPlaceholder />
</div>
</div>
</div>
@ -224,6 +216,22 @@
<div class="w-full">
{#if prompt.charAt(0) === '/'}
<Prompts bind:this={promptsElement} bind:prompt />
{:else if prompt.charAt(0) === '#'}
<Documents
bind:this={documentsElement}
bind:prompt
on:select={(e) => {
console.log(e);
files = [
...files,
{
type: 'doc',
...e.detail,
upload_status: true
}
];
}}
/>
{:else if messages.length == 0 && suggestionPrompts.length !== 0}
<Suggestions {suggestionPrompts} {submitPrompt} />
{/if}
@ -256,14 +264,7 @@
const file = inputFiles[0];
if (['image/gif', 'image/jpeg', 'image/png'].includes(file['type'])) {
reader.readAsDataURL(file);
} else if (
[
'application/pdf',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'text/plain',
'text/csv'
].includes(file['type'])
) {
} else if (SUPPORTED_FILE_TYPE.includes(file['type'])) {
uploadDoc(file);
filesInputElement.value = '';
} else {
@ -448,8 +449,10 @@
editButton?.click();
}
if (prompt.charAt(0) === '/' && e.key === 'ArrowUp') {
promptsElement.selectUp();
if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'ArrowUp') {
e.preventDefault();
(promptsElement || documentsElement).selectUp();
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
@ -457,8 +460,10 @@
commandOptionButton.scrollIntoView({ block: 'center' });
}
if (prompt.charAt(0) === '/' && e.key === 'ArrowDown') {
promptsElement.selectDown();
if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'ArrowDown') {
e.preventDefault();
(promptsElement || documentsElement).selectDown();
const commandOptionButton = [
...document.getElementsByClassName('selected-command-option-button')
@ -466,7 +471,7 @@
commandOptionButton.scrollIntoView({ block: 'center' });
}
if (prompt.charAt(0) === '/' && e.key === 'Enter') {
if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'Enter') {
e.preventDefault();
const commandOptionButton = [
@ -476,7 +481,7 @@
commandOptionButton?.click();
}
if (prompt.charAt(0) === '/' && e.key === 'Tab') {
if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'Tab') {
e.preventDefault();
const commandOptionButton = [

View file

@ -0,0 +1,78 @@
<script lang="ts">
import { createEventDispatcher } from 'svelte';
import { documents } from '$lib/stores';
import { removeFirstHashWord } from '$lib/utils';
import { tick } from 'svelte';
export let prompt = '';
const dispatch = createEventDispatcher();
let selectedIdx = 0;
let filteredDocs = [];
$: filteredDocs = $documents
.filter((p) => p.name.includes(prompt.split(' ')?.at(0)?.substring(1) ?? ''))
.sort((a, b) => a.title.localeCompare(b.title));
$: if (prompt) {
selectedIdx = 0;
}
export const selectUp = () => {
selectedIdx = Math.max(0, selectedIdx - 1);
};
export const selectDown = () => {
selectedIdx = Math.min(selectedIdx + 1, filteredDocs.length - 1);
};
const confirmSelect = async (doc) => {
dispatch('select', doc);
prompt = removeFirstHashWord(prompt);
const chatInputElement = document.getElementById('chat-textarea');
await tick();
chatInputElement?.focus();
await tick();
};
</script>
{#if filteredDocs.length > 0}
<div class="md:px-2 mb-3 text-left w-full">
<div class="flex w-full rounded-lg border border-gray-100 dark:border-gray-700">
<div class=" bg-gray-100 dark:bg-gray-700 w-10 rounded-l-lg text-center">
<div class=" text-lg font-semibold mt-2">#</div>
</div>
<div class="max-h-60 flex flex-col w-full rounded-r-lg">
<div class=" overflow-y-auto bg-white p-2 rounded-tr-lg space-y-0.5">
{#each filteredDocs as doc, docIdx}
<button
class=" px-3 py-1.5 rounded-lg w-full text-left {docIdx === selectedIdx
? ' bg-gray-100 selected-command-option-button'
: ''}"
type="button"
on:click={() => {
confirmSelect(doc);
}}
on:mousemove={() => {
selectedIdx = docIdx;
}}
on:focus={() => {}}
>
<div class=" font-medium text-black line-clamp-1">
#{doc.name} ({doc.filename})
</div>
<div class=" text-xs text-gray-600 line-clamp-1">
{doc.title}
</div>
</button>
{/each}
</div>
</div>
</div>
</div>
{/if}

View file

@ -11,6 +11,13 @@ export const WEB_UI_VERSION = 'v1.0.0-alpha-static';
export const REQUIRED_OLLAMA_VERSION = '0.1.16';
export const SUPPORTED_FILE_TYPE = [
'application/pdf',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'text/plain',
'text/csv'
];
// Source: https://kit.svelte.dev/docs/modules#$env-static-public
// This feature, akin to $env/static/private, exclusively incorporates environment variables
// that are prefixed with config.kit.env.publicPrefix (usually set to PUBLIC_).

View file

@ -11,8 +11,23 @@ export const chatId = writable('');
export const chats = writable([]);
export const models = writable([]);
export const modelfiles = writable([]);
export const prompts = writable([]);
export const documents = writable([
{
collection_name: 'collection_name',
filename: 'filename',
name: 'name',
title: 'title'
},
{
collection_name: 'collection_name1',
filename: 'filename1',
name: 'name1',
title: 'title1'
}
]);
export const settings = writable({});
export const showSettings = writable(false);

View file

@ -128,6 +128,24 @@ export const findWordIndices = (text) => {
return matches;
};
export const removeFirstHashWord = (inputString) => {
// Split the string into an array of words
const words = inputString.split(' ');
// Find the index of the first word that starts with #
const index = words.findIndex((word) => word.startsWith('#'));
// Remove the first word with #
if (index !== -1) {
words.splice(index, 1);
}
// Join the remaining words back into a string
const resultString = words.join(' ');
return resultString;
};
export const calculateSHA256 = async (file) => {
// Create a FileReader to read the file asynchronously
const reader = new FileReader();

View file

@ -0,0 +1,306 @@
<script lang="ts">
import toast from 'svelte-french-toast';
import fileSaver from 'file-saver';
const { saveAs } = fileSaver;
import { onMount } from 'svelte';
import { documents } from '$lib/stores';
import { createNewDoc, deleteDocByName, getDocs } from '$lib/apis/documents';
import AddFilesPlaceholder from '$lib/components/AddFilesPlaceholder.svelte';
import { SUPPORTED_FILE_TYPE } from '$lib/constants';
let importFiles = '';
let query = '';
let dragged = false;
const deleteDoc = async (name) => {
await deleteDocByName(localStorage.token, name);
await documents.set(await getDocs(localStorage.token));
};
onMount(() => {
// const dropZone = document.querySelector('body');
const dropZone = document.getElementById('dropzone');
dropZone?.addEventListener('dragover', (e) => {
e.preventDefault();
dragged = true;
});
dropZone?.addEventListener('drop', async (e) => {
e.preventDefault();
console.log(e);
if (e.dataTransfer?.files) {
const inputFiles = e.dataTransfer?.files;
if (inputFiles && inputFiles.length > 0) {
const file = inputFiles[0];
if (SUPPORTED_FILE_TYPE.includes(file['type'])) {
console.log(file);
// uploadDoc(file);
} else {
toast.error(`Unsupported File Type '${file['type']}'.`);
}
} else {
toast.error(`File not found.`);
}
}
});
dropZone?.addEventListener('dragleave', () => {
dragged = false;
});
});
</script>
<div class="min-h-screen w-full flex justify-center dark:text-white">
<div class=" py-2.5 flex flex-col justify-between w-full">
<div class="max-w-2xl mx-auto w-full px-3 md:px-0 my-10">
<div class="mb-6 flex justify-between items-center">
<div class=" text-2xl font-semibold self-center">My Documents</div>
</div>
<div class=" flex w-full space-x-2">
<div class="flex flex-1">
<div class=" self-center ml-1 mr-3">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M9 3.5a5.5 5.5 0 100 11 5.5 5.5 0 000-11zM2 9a7 7 0 1112.452 4.391l3.328 3.329a.75.75 0 11-1.06 1.06l-3.329-3.328A7 7 0 012 9z"
clip-rule="evenodd"
/>
</svg>
</div>
<input
class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-none bg-transparent"
bind:value={query}
placeholder="Search Document"
/>
</div>
<div>
<a
class=" px-2 py-2 rounded-xl border border-gray-200 dark:border-gray-600 hover:bg-gray-100 dark:bg-gray-800 dark:hover:bg-gray-700 transition font-medium text-sm flex items-center space-x-1"
href="/prompts/create"
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
/>
</svg>
</a>
</div>
</div>
<div
class="z-50 touch-none pointer-events-none"
id="dropzone"
role="region"
aria-label="Drag and Drop Container"
>
{#if $documents.length === 0 || dragged}
<div class="my-3 py-16 rounded-lg border-2 border-dashed dark:border-gray-600">
<AddFilesPlaceholder />
</div>
{:else}
{#each $documents.filter((p) => query === '' || p.name.includes(query)) as doc}
<hr class=" dark:border-gray-700 my-2.5" />
<div class=" flex space-x-4 cursor-pointer w-full mb-3">
<div class=" flex flex-1 space-x-4 cursor-pointer w-full">
<a href={`/prompts/edit?command=${encodeURIComponent(doc.name)}`}>
<div class=" flex-1 self-center pl-5">
<div class=" font-bold line-clamp-1">#{doc.name} ({doc.filename})</div>
<div class=" text-xs overflow-hidden text-ellipsis line-clamp-1">
{doc.title}
</div>
</div>
</a>
</div>
<div class="flex flex-row space-x-1 self-center">
<a
class="self-center w-fit text-sm px-2 py-2 border dark:border-gray-600 rounded-xl"
type="button"
href={`/prompts/edit?command=${encodeURIComponent(doc.name)}`}
>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="w-4 h-4"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M16.862 4.487l1.687-1.688a1.875 1.875 0 112.652 2.652L6.832 19.82a4.5 4.5 0 01-1.897 1.13l-2.685.8.8-2.685a4.5 4.5 0 011.13-1.897L16.863 4.487zm0 0L19.5 7.125"
/>
</svg>
</a>
<!-- <button
class="self-center w-fit text-sm px-2 py-2 border dark:border-gray-600 rounded-xl"
type="button"
on:click={() => {
sharePrompt(prompt);
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="w-4 h-4"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M7.217 10.907a2.25 2.25 0 100 2.186m0-2.186c.18.324.283.696.283 1.093s-.103.77-.283 1.093m0-2.186l9.566-5.314m-9.566 7.5l9.566 5.314m0 0a2.25 2.25 0 103.935 2.186 2.25 2.25 0 00-3.935-2.186zm0-12.814a2.25 2.25 0 103.933-2.185 2.25 2.25 0 00-3.933 2.185z"
/>
</svg>
</button> -->
<button
class="self-center w-fit text-sm px-2 py-2 border dark:border-gray-600 rounded-xl"
type="button"
on:click={() => {
deleteDoc(doc.name);
}}
>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="w-4 h-4"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M14.74 9l-.346 9m-4.788 0L9.26 9m9.968-3.21c.342.052.682.107 1.022.166m-1.022-.165L18.16 19.673a2.25 2.25 0 01-2.244 2.077H8.084a2.25 2.25 0 01-2.244-2.077L4.772 5.79m14.456 0a48.108 48.108 0 00-3.478-.397m-12 .562c.34-.059.68-.114 1.022-.165m0 0a48.11 48.11 0 013.478-.397m7.5 0v-.916c0-1.18-.91-2.164-2.09-2.201a51.964 51.964 0 00-3.32 0c-1.18.037-2.09 1.022-2.09 2.201v.916m7.5 0a48.667 48.667 0 00-7.5 0"
/>
</svg>
</button>
</div>
</div>
{/each}
{/if}
</div>
{#if $documents.length != 0}
<hr class=" dark:border-gray-700 my-2.5" />
<div class=" flex justify-between w-full mb-3">
<div class="flex space-x-2">
<input
id="documents-import-input"
bind:files={importFiles}
type="file"
accept=".json"
hidden
on:change={() => {
console.log(importFiles);
const reader = new FileReader();
reader.onload = async (event) => {
const savedDocs = JSON.parse(event.target.result);
console.log(savedDocs);
for (const doc of savedDocs) {
await createNewDoc(
localStorage.token,
doc.collection_name,
doc.filename,
doc.name,
doc.title
).catch((error) => {
toast.error(error);
return null;
});
}
await documents.set(await getDocs(localStorage.token));
};
reader.readAsText(importFiles[0]);
}}
/>
<button
class="self-center w-fit text-sm px-3 py-1 border dark:border-gray-600 rounded-xl flex"
on:click={async () => {
document.getElementById('documents-import-input')?.click();
}}
>
<div class=" self-center mr-2 font-medium">Import Documents Mapping</div>
<div class=" self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M4 2a1.5 1.5 0 0 0-1.5 1.5v9A1.5 1.5 0 0 0 4 14h8a1.5 1.5 0 0 0 1.5-1.5V6.621a1.5 1.5 0 0 0-.44-1.06L9.94 2.439A1.5 1.5 0 0 0 8.878 2H4Zm4 9.5a.75.75 0 0 1-.75-.75V8.06l-.72.72a.75.75 0 0 1-1.06-1.06l2-2a.75.75 0 0 1 1.06 0l2 2a.75.75 0 1 1-1.06 1.06l-.72-.72v2.69a.75.75 0 0 1-.75.75Z"
clip-rule="evenodd"
/>
</svg>
</div>
</button>
<button
class="self-center w-fit text-sm px-3 py-1 border dark:border-gray-600 rounded-xl flex"
on:click={async () => {
let blob = new Blob([JSON.stringify($documents)], {
type: 'application/json'
});
saveAs(blob, `documents-mapping-export-${Date.now()}.json`);
}}
>
<div class=" self-center mr-2 font-medium">Export Documents Mapping</div>
<div class=" self-center">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 16 16"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M4 2a1.5 1.5 0 0 0-1.5 1.5v9A1.5 1.5 0 0 0 4 14h8a1.5 1.5 0 0 0 1.5-1.5V6.621a1.5 1.5 0 0 0-.44-1.06L9.94 2.439A1.5 1.5 0 0 0 8.878 2H4Zm4 3.5a.75.75 0 0 1 .75.75v2.69l.72-.72a.75.75 0 1 1 1.06 1.06l-2 2a.75.75 0 0 1-1.06 0l-2-2a.75.75 0 0 1 1.06-1.06l.72.72V6.25A.75.75 0 0 1 8 5.5Z"
clip-rule="evenodd"
/>
</svg>
</div>
</button>
<!-- <button
on:click={() => {
loadDefaultPrompts();
}}
>
dd
</button> -->
</div>
</div>
{/if}
</div>
</div>
</div>