From 2603ac30bc4dc04069759cd1d5ce9eb2f747cd9e Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sun, 7 Jan 2024 22:14:08 -0800 Subject: [PATCH 1/9] feat: documents --- backend/config.py | 2 +- src/lib/components/layout/Sidebar.svelte | 50 +++++++++++++++++++----- src/routes/(app)/documents/+page.svelte | 0 3 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 src/routes/(app)/documents/+page.svelte diff --git a/backend/config.py b/backend/config.py index 03718a06..2a96d018 100644 --- a/backend/config.py +++ b/backend/config.py @@ -58,7 +58,7 @@ if OPENAI_API_BASE_URL == "": # WEBUI_VERSION #################################### -WEBUI_VERSION = os.environ.get("WEBUI_VERSION", "v1.0.0-alpha.50") +WEBUI_VERSION = os.environ.get("WEBUI_VERSION", "v1.0.0-alpha.61") #################################### # WEBUI_AUTH (Required for security) diff --git a/src/lib/components/layout/Sidebar.svelte b/src/lib/components/layout/Sidebar.svelte index 7ea08f98..927d87d6 100644 --- a/src/lib/components/layout/Sidebar.svelte +++ b/src/lib/components/layout/Sidebar.svelte @@ -68,7 +68,7 @@
{#if $user?.role === 'admin'} -
+
@@ -136,7 +136,7 @@
-
+
+ +
+ +
{/if}
diff --git a/src/routes/(app)/documents/+page.svelte b/src/routes/(app)/documents/+page.svelte new file mode 100644 index 00000000..e69de29b From cc3f84f916ccbff3bbd6ff794c92ddcab910ea08 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sun, 7 Jan 2024 23:43:32 -0800 Subject: [PATCH 2/9] feat: # to import doc --- backend/apps/web/main.py | 16 +- backend/apps/web/models/documents.py | 123 +++++++ backend/apps/web/routers/documents.py | 119 +++++++ src/lib/apis/documents/index.ts | 177 ++++++++++ src/lib/components/AddFilesPlaceholder.svelte | 6 + src/lib/components/chat/MessageInput.svelte | 61 ++-- .../chat/MessageInput/Documents.svelte | 78 +++++ src/lib/constants.ts | 7 + src/lib/stores/index.ts | 15 + src/lib/utils/index.ts | 18 ++ src/routes/(app)/documents/+page.svelte | 306 ++++++++++++++++++ 11 files changed, 894 insertions(+), 32 deletions(-) create mode 100644 backend/apps/web/models/documents.py create mode 100644 backend/apps/web/routers/documents.py create mode 100644 src/lib/apis/documents/index.ts create mode 100644 src/lib/components/AddFilesPlaceholder.svelte create mode 100644 src/lib/components/chat/MessageInput/Documents.svelte diff --git a/backend/apps/web/main.py b/backend/apps/web/main.py index 0030c25e..89616064 100644 --- a/backend/apps/web/main.py +++ b/backend/apps/web/main.py @@ -1,7 +1,16 @@ from fastapi import FastAPI, Depends from fastapi.routing import APIRoute from fastapi.middleware.cors import CORSMiddleware -from apps.web.routers import auths, users, chats, modelfiles, prompts, configs, utils +from apps.web.routers import ( + auths, + users, + chats, + documents, + modelfiles, + prompts, + configs, + utils, +) from config import WEBUI_VERSION, WEBUI_AUTH app = FastAPI() @@ -22,9 +31,8 @@ app.add_middleware( app.include_router(auths.router, prefix="/auths", tags=["auths"]) app.include_router(users.router, prefix="/users", tags=["users"]) app.include_router(chats.router, prefix="/chats", tags=["chats"]) -app.include_router(modelfiles.router, - prefix="/modelfiles", - tags=["modelfiles"]) +app.include_router(documents.router, prefix="/documents", tags=["documents"]) +app.include_router(modelfiles.router, prefix="/modelfiles", tags=["modelfiles"]) app.include_router(prompts.router, prefix="/prompts", tags=["prompts"]) app.include_router(configs.router, prefix="/configs", tags=["configs"]) diff --git a/backend/apps/web/models/documents.py b/backend/apps/web/models/documents.py new file mode 100644 index 00000000..ccd76194 --- /dev/null +++ b/backend/apps/web/models/documents.py @@ -0,0 +1,123 @@ +from pydantic import BaseModel +from peewee import * +from playhouse.shortcuts import model_to_dict +from typing import List, Union, Optional +import time + +from utils.utils import decode_token +from utils.misc import get_gravatar_url + +from apps.web.internal.db import DB + +import json + +#################### +# Documents DB Schema +#################### + + +class Document(Model): + collection_name = CharField(unique=True) + name = CharField(unique=True) + title = CharField() + filename = CharField() + content = TextField(null=True) + user_id = CharField() + timestamp = DateField() + + class Meta: + database = DB + + +class DocumentModel(BaseModel): + collection_name: str + name: str + title: str + filename: str + content: Optional[str] = None + user_id: str + timestamp: int # timestamp in epoch + + +#################### +# Forms +#################### + + +class DocumentUpdateForm(BaseModel): + name: str + title: str + + +class DocumentForm(DocumentUpdateForm): + collection_name: str + filename: str + content: Optional[str] = None + + +class DocumentsTable: + def __init__(self, db): + self.db = db + self.db.create_tables([Document]) + + def insert_new_doc( + self, user_id: str, form_data: DocumentForm + ) -> Optional[DocumentModel]: + document = DocumentModel( + **{ + **form_data.model_dump(), + "user_id": user_id, + "timestamp": int(time.time()), + } + ) + + try: + result = Document.create(**document.model_dump()) + if result: + return document + else: + return None + except: + return None + + def get_doc_by_name(self, name: str) -> Optional[DocumentModel]: + try: + document = Document.get(Document.name == name) + return DocumentModel(**model_to_dict(document)) + except: + return None + + def get_docs(self) -> List[DocumentModel]: + return [ + DocumentModel(**model_to_dict(doc)) + for doc in Document.select() + # .limit(limit).offset(skip) + ] + + def update_doc_by_name( + self, name: str, form_data: DocumentUpdateForm + ) -> Optional[DocumentModel]: + try: + query = Document.update( + title=form_data.title, + name=form_data.name, + timestamp=int(time.time()), + ).where(Document.name == name) + query.execute() + + doc = Document.get(Document.name == name) + return DocumentModel(**model_to_dict(doc)) + except: + return None + + def delete_doc_by_name(self, name: str) -> bool: + try: + query = Document.delete().where((Document.name == name)) + query.execute() # Remove the rows, return number of rows removed. + + return True + except: + return False + + +Documents = DocumentsTable(DB) diff --git a/backend/apps/web/routers/documents.py b/backend/apps/web/routers/documents.py new file mode 100644 index 00000000..42463f51 --- /dev/null +++ b/backend/apps/web/routers/documents.py @@ -0,0 +1,119 @@ +from fastapi import Depends, FastAPI, HTTPException, status +from datetime import datetime, timedelta +from typing import List, Union, Optional + +from fastapi import APIRouter +from pydantic import BaseModel +import json + +from apps.web.models.documents import ( + Documents, + DocumentForm, + DocumentUpdateForm, + DocumentModel, +) + +from utils.utils import get_current_user +from constants import ERROR_MESSAGES + +router = APIRouter() + +############################ +# GetDocuments +############################ + + +@router.get("/", response_model=List[DocumentModel]) +async def get_documents(user=Depends(get_current_user)): + return Documents.get_docs() + + +############################ +# CreateNewDoc +############################ + + +@router.post("/create", response_model=Optional[DocumentModel]) +async def create_new_doc(form_data: DocumentForm, user=Depends(get_current_user)): + if user.role != "admin": + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=ERROR_MESSAGES.ACCESS_PROHIBITED, + ) + + doc = Documents.get_doc_by_name(form_data.name) + if doc == None: + doc = Documents.insert_new_doc(user.id, form_data) + + if doc: + return doc + else: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=ERROR_MESSAGES.DEFAULT(), + ) + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.COMMAND_TAKEN, + ) + + +############################ +# GetDocByName +############################ + + +@router.get("/name/{name}", response_model=Optional[DocumentModel]) +async def get_doc_by_name(name: str, user=Depends(get_current_user)): + doc = Documents.get_doc_by_name(name) + + if doc: + return doc + else: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=ERROR_MESSAGES.NOT_FOUND, + ) + + +############################ +# UpdateDocByName +############################ + + +@router.post("/name/{name}/update", response_model=Optional[DocumentModel]) +async def update_doc_by_name( + name: str, form_data: DocumentUpdateForm, user=Depends(get_current_user) +): + if user.role != "admin": + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=ERROR_MESSAGES.ACCESS_PROHIBITED, + ) + + doc = Documents.update_doc_by_name(name, form_data) + if doc: + return doc + else: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=ERROR_MESSAGES.ACCESS_PROHIBITED, + ) + + +############################ +# DeleteDocByName +############################ + + +@router.delete("/name/{name}/delete", response_model=bool) +async def delete_doc_by_name(name: str, user=Depends(get_current_user)): + if user.role != "admin": + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=ERROR_MESSAGES.ACCESS_PROHIBITED, + ) + + result = Documents.delete_doc_by_name(name) + return result diff --git a/src/lib/apis/documents/index.ts b/src/lib/apis/documents/index.ts new file mode 100644 index 00000000..37bdde04 --- /dev/null +++ b/src/lib/apis/documents/index.ts @@ -0,0 +1,177 @@ +import { WEBUI_API_BASE_URL } from '$lib/constants'; + +export const createNewDoc = async ( + token: string, + collection_name: string, + filename: string, + name: string, + title: string +) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/documents/create`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + collection_name: collection_name, + filename: filename, + name: name, + title: title + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const getDocs = async (token: string = '') => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/documents/`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const getDocByName = async (token: string, name: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/documents/name/${name}`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +type DocUpdateForm = { + name: string; + title: string; +}; + +export const updateDocByName = async (token: string, name: string, form: DocUpdateForm) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/prompts/name/${name}/update`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + name: form.name, + title: form.title + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const deleteDocByName = async (token: string, name: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/documents/name/${name}/delete`, { + method: 'DELETE', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; diff --git a/src/lib/components/AddFilesPlaceholder.svelte b/src/lib/components/AddFilesPlaceholder.svelte new file mode 100644 index 00000000..7cc51c6f --- /dev/null +++ b/src/lib/components/AddFilesPlaceholder.svelte @@ -0,0 +1,6 @@ +
📄
+
Add Files
+ +
+ Drop any files here to add to the conversation +
diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index 54ccc8f4..c81d999c 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -7,6 +7,9 @@ import Prompts from './MessageInput/PromptCommands.svelte'; import Suggestions from './MessageInput/Suggestions.svelte'; import { uploadDocToVectorDB } from '$lib/apis/rag'; + import AddFilesPlaceholder from '../AddFilesPlaceholder.svelte'; + import { SUPPORTED_FILE_TYPE } from '$lib/constants'; + import Documents from './MessageInput/Documents.svelte'; export let submitPrompt: Function; export let stopResponse: Function; @@ -16,6 +19,7 @@ let filesInputElement; let promptsElement; + let documentsElement; let inputFiles; let dragged = false; @@ -143,14 +147,7 @@ const file = inputFiles[0]; if (['image/gif', 'image/jpeg', 'image/png'].includes(file['type'])) { reader.readAsDataURL(file); - } else if ( - [ - 'application/pdf', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'text/plain', - 'text/csv' - ].includes(file['type']) - ) { + } else if (SUPPORTED_FILE_TYPE.includes(file['type'])) { uploadDoc(file); } else { toast.error(`Unsupported File Type '${file['type']}'.`); @@ -179,12 +176,7 @@
-
🗂️
-
Add Files
- -
- Drop any files/images here to add to the conversation -
+
@@ -224,6 +216,22 @@
{#if prompt.charAt(0) === '/'} + {:else if prompt.charAt(0) === '#'} + { + console.log(e); + files = [ + ...files, + { + type: 'doc', + ...e.detail, + upload_status: true + } + ]; + }} + /> {:else if messages.length == 0 && suggestionPrompts.length !== 0} {/if} @@ -256,14 +264,7 @@ const file = inputFiles[0]; if (['image/gif', 'image/jpeg', 'image/png'].includes(file['type'])) { reader.readAsDataURL(file); - } else if ( - [ - 'application/pdf', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'text/plain', - 'text/csv' - ].includes(file['type']) - ) { + } else if (SUPPORTED_FILE_TYPE.includes(file['type'])) { uploadDoc(file); filesInputElement.value = ''; } else { @@ -448,8 +449,10 @@ editButton?.click(); } - if (prompt.charAt(0) === '/' && e.key === 'ArrowUp') { - promptsElement.selectUp(); + if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'ArrowUp') { + e.preventDefault(); + + (promptsElement || documentsElement).selectUp(); const commandOptionButton = [ ...document.getElementsByClassName('selected-command-option-button') @@ -457,8 +460,10 @@ commandOptionButton.scrollIntoView({ block: 'center' }); } - if (prompt.charAt(0) === '/' && e.key === 'ArrowDown') { - promptsElement.selectDown(); + if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'ArrowDown') { + e.preventDefault(); + + (promptsElement || documentsElement).selectDown(); const commandOptionButton = [ ...document.getElementsByClassName('selected-command-option-button') @@ -466,7 +471,7 @@ commandOptionButton.scrollIntoView({ block: 'center' }); } - if (prompt.charAt(0) === '/' && e.key === 'Enter') { + if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'Enter') { e.preventDefault(); const commandOptionButton = [ @@ -476,7 +481,7 @@ commandOptionButton?.click(); } - if (prompt.charAt(0) === '/' && e.key === 'Tab') { + if (['/', '#'].includes(prompt.charAt(0)) && e.key === 'Tab') { e.preventDefault(); const commandOptionButton = [ diff --git a/src/lib/components/chat/MessageInput/Documents.svelte b/src/lib/components/chat/MessageInput/Documents.svelte new file mode 100644 index 00000000..bcfb1916 --- /dev/null +++ b/src/lib/components/chat/MessageInput/Documents.svelte @@ -0,0 +1,78 @@ + + +{#if filteredDocs.length > 0} +
+
+
+
#
+
+ +
+
+ {#each filteredDocs as doc, docIdx} + + {/each} +
+
+
+
+{/if} diff --git a/src/lib/constants.ts b/src/lib/constants.ts index a43104ad..260e675e 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -11,6 +11,13 @@ export const WEB_UI_VERSION = 'v1.0.0-alpha-static'; export const REQUIRED_OLLAMA_VERSION = '0.1.16'; +export const SUPPORTED_FILE_TYPE = [ + 'application/pdf', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'text/plain', + 'text/csv' +]; + // Source: https://kit.svelte.dev/docs/modules#$env-static-public // This feature, akin to $env/static/private, exclusively incorporates environment variables // that are prefixed with config.kit.env.publicPrefix (usually set to PUBLIC_). diff --git a/src/lib/stores/index.ts b/src/lib/stores/index.ts index 55c83b25..c7d8f5e6 100644 --- a/src/lib/stores/index.ts +++ b/src/lib/stores/index.ts @@ -11,8 +11,23 @@ export const chatId = writable(''); export const chats = writable([]); export const models = writable([]); + export const modelfiles = writable([]); export const prompts = writable([]); +export const documents = writable([ + { + collection_name: 'collection_name', + filename: 'filename', + name: 'name', + title: 'title' + }, + { + collection_name: 'collection_name1', + filename: 'filename1', + name: 'name1', + title: 'title1' + } +]); export const settings = writable({}); export const showSettings = writable(false); diff --git a/src/lib/utils/index.ts b/src/lib/utils/index.ts index 46bc8f04..75cd073e 100644 --- a/src/lib/utils/index.ts +++ b/src/lib/utils/index.ts @@ -128,6 +128,24 @@ export const findWordIndices = (text) => { return matches; }; +export const removeFirstHashWord = (inputString) => { + // Split the string into an array of words + const words = inputString.split(' '); + + // Find the index of the first word that starts with # + const index = words.findIndex((word) => word.startsWith('#')); + + // Remove the first word with # + if (index !== -1) { + words.splice(index, 1); + } + + // Join the remaining words back into a string + const resultString = words.join(' '); + + return resultString; +}; + export const calculateSHA256 = async (file) => { // Create a FileReader to read the file asynchronously const reader = new FileReader(); diff --git a/src/routes/(app)/documents/+page.svelte b/src/routes/(app)/documents/+page.svelte index e69de29b..c28390f1 100644 --- a/src/routes/(app)/documents/+page.svelte +++ b/src/routes/(app)/documents/+page.svelte @@ -0,0 +1,306 @@ + + +
+
+
+
+
My Documents
+
+ +
+
+
+ + + +
+ +
+ + +
+ +
+ {#if $documents.length === 0 || dragged} +
+ +
+ {:else} + {#each $documents.filter((p) => query === '' || p.name.includes(query)) as doc} +
+
+
+ +
+
#{doc.name} ({doc.filename})
+
+ {doc.title} +
+
+ +
+
+ + + + + + + + + +
+
+ {/each} + {/if} +
+ + {#if $documents.length != 0} +
+ +
+
+ { + console.log(importFiles); + + const reader = new FileReader(); + reader.onload = async (event) => { + const savedDocs = JSON.parse(event.target.result); + console.log(savedDocs); + + for (const doc of savedDocs) { + await createNewDoc( + localStorage.token, + doc.collection_name, + doc.filename, + doc.name, + doc.title + ).catch((error) => { + toast.error(error); + return null; + }); + } + + await documents.set(await getDocs(localStorage.token)); + }; + + reader.readAsText(importFiles[0]); + }} + /> + + + + + + +
+
+ {/if} +
+
+
From df3d95bf2a445ac0bac8d8235da9d4402f0985d8 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Mon, 8 Jan 2024 01:12:02 -0800 Subject: [PATCH 3/9] refac: message drag file input --- src/lib/components/chat/MessageInput.svelte | 24 +++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index c81d999c..34f4de18 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -119,12 +119,16 @@ onMount(() => { const dropZone = document.querySelector('body'); - dropZone?.addEventListener('dragover', (e) => { + const onDragOver = (e) => { e.preventDefault(); dragged = true; - }); + }; - dropZone.addEventListener('drop', async (e) => { + const onDragLeave = () => { + dragged = false; + }; + + const onDrop = async (e) => { e.preventDefault(); console.log(e); @@ -158,11 +162,17 @@ } dragged = false; - }); + }; - dropZone?.addEventListener('dragleave', () => { - dragged = false; - }); + dropZone?.addEventListener('dragover', onDragOver); + dropZone?.addEventListener('drop', onDrop); + dropZone?.addEventListener('dragleave', onDragLeave); + + return () => { + dropZone?.removeEventListener('dragover', onDragOver); + dropZone?.removeEventListener('drop', onDrop); + dropZone?.removeEventListener('dragleave', onDragLeave); + }; }); From 54c4e0761a5b9e5d102c1c0dface786cf489fdd7 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Mon, 8 Jan 2024 01:26:15 -0800 Subject: [PATCH 4/9] feat: documents file upload --- backend/apps/rag/main.py | 12 +- src/routes/(app)/+layout.svelte | 16 +- src/routes/(app)/documents/+page.svelte | 262 ++++++++++++++---------- 3 files changed, 174 insertions(+), 116 deletions(-) diff --git a/backend/apps/rag/main.py b/backend/apps/rag/main.py index 6e4f5c09..0d6cc732 100644 --- a/backend/apps/rag/main.py +++ b/backend/apps/rag/main.py @@ -119,7 +119,11 @@ def store_web(form_data: StoreWebForm, user=Depends(get_current_user)): loader = WebBaseLoader(form_data.url) data = loader.load() store_data_in_vector_db(data, form_data.collection_name) - return {"status": True, "collection_name": form_data.collection_name} + return { + "status": True, + "collection_name": form_data.collection_name, + "filename": form_data.url, + } except Exception as e: print(e) raise HTTPException( @@ -176,7 +180,11 @@ def store_doc( result = store_data_in_vector_db(data, collection_name) if result: - return {"status": True, "collection_name": collection_name} + return { + "status": True, + "collection_name": collection_name, + "filename": filename, + } else: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/src/routes/(app)/+layout.svelte b/src/routes/(app)/+layout.svelte index c264592e..39ae0eea 100644 --- a/src/routes/(app)/+layout.svelte +++ b/src/routes/(app)/+layout.svelte @@ -13,13 +13,22 @@ import { getOpenAIModels } from '$lib/apis/openai'; - import { user, showSettings, settings, models, modelfiles, prompts } from '$lib/stores'; + import { + user, + showSettings, + settings, + models, + modelfiles, + prompts, + documents + } from '$lib/stores'; import { REQUIRED_OLLAMA_VERSION, WEBUI_API_BASE_URL } from '$lib/constants'; import SettingsModal from '$lib/components/chat/SettingsModal.svelte'; import Sidebar from '$lib/components/layout/Sidebar.svelte'; import { checkVersion } from '$lib/utils'; import ShortcutsModal from '$lib/components/chat/ShortcutsModal.svelte'; + import { getDocs } from '$lib/apis/documents'; let ollamaVersion = ''; let loaded = false; @@ -93,11 +102,10 @@ console.log(); await settings.set(JSON.parse(localStorage.getItem('settings') ?? '{}')); + await modelfiles.set(await getModelfiles(localStorage.token)); - await prompts.set(await getPrompts(localStorage.token)); - - console.log($modelfiles); + await documents.set(await getDocs(localStorage.token)); modelfiles.subscribe(async () => { // should fetch models diff --git a/src/routes/(app)/documents/+page.svelte b/src/routes/(app)/documents/+page.svelte index c28390f1..747adec8 100644 --- a/src/routes/(app)/documents/+page.svelte +++ b/src/routes/(app)/documents/+page.svelte @@ -6,10 +6,13 @@ import { onMount } from 'svelte'; import { documents } from '$lib/stores'; import { createNewDoc, deleteDocByName, getDocs } from '$lib/apis/documents'; - import AddFilesPlaceholder from '$lib/components/AddFilesPlaceholder.svelte'; + import { SUPPORTED_FILE_TYPE } from '$lib/constants'; + import { uploadDocToVectorDB } from '$lib/apis/rag'; let importFiles = ''; + + let inputFiles = ''; let query = ''; let dragged = false; @@ -19,40 +22,51 @@ await documents.set(await getDocs(localStorage.token)); }; - onMount(() => { - // const dropZone = document.querySelector('body'); - const dropZone = document.getElementById('dropzone'); + const uploadDoc = async (file) => { + const res = await uploadDocToVectorDB(localStorage.token, '', file); - dropZone?.addEventListener('dragover', (e) => { - e.preventDefault(); - dragged = true; - }); + if (res) { + await createNewDoc( + localStorage.token, + res.collection_name, + res.filename, + res.filename, + res.filename + ); + await documents.set(await getDocs(localStorage.token)); + } + }; - dropZone?.addEventListener('drop', async (e) => { - e.preventDefault(); - console.log(e); + const onDragOver = (e) => { + e.preventDefault(); + dragged = true; + }; - if (e.dataTransfer?.files) { - const inputFiles = e.dataTransfer?.files; + const onDragLeave = () => { + dragged = false; + }; - if (inputFiles && inputFiles.length > 0) { - const file = inputFiles[0]; - if (SUPPORTED_FILE_TYPE.includes(file['type'])) { - console.log(file); - // uploadDoc(file); - } else { - toast.error(`Unsupported File Type '${file['type']}'.`); - } + const onDrop = async (e) => { + e.preventDefault(); + console.log(e); + + if (e.dataTransfer?.files) { + const inputFiles = e.dataTransfer?.files; + + if (inputFiles && inputFiles.length > 0) { + const file = inputFiles[0]; + if (SUPPORTED_FILE_TYPE.includes(file['type'])) { + uploadDoc(file); } else { - toast.error(`File not found.`); + toast.error(`Unsupported File Type '${file['type']}'.`); } + } else { + toast.error(`File not found.`); } - }); + } - dropZone?.addEventListener('dragleave', () => { - dragged = false; - }); - }); + dragged = false; + };
@@ -86,9 +100,11 @@
-
- {#if $documents.length === 0 || dragged} -
- + { + if (inputFiles && inputFiles.length > 0) { + const file = inputFiles[0]; + if (SUPPORTED_FILE_TYPE.includes(file['type'])) { + uploadDoc(file); + } else { + toast.error(`Unsupported File Type '${file['type']}'.`); + } + + inputFiles = null; + e.target.value = ''; + } else { + toast.error(`File not found.`); + } + }} + /> + +
+
+
+
Add Files
+ +
+ Drop any files here to add to the conversation +
- {:else} - {#each $documents.filter((p) => query === '' || p.name.includes(query)) as doc} -
-
-
- -
-
#{doc.name} ({doc.filename})
-
- {doc.title} -
-
- +
+
+ + {#each $documents.filter((p) => query === '' || p.name.includes(query)) as doc} +
+
+
+ +
+
#{doc.name} ({doc.filename})
+
+ {doc.title} +
-
- - - - - - - + + + - -
-
- {/each} - {/if} -
+ --> + +
+
+ {/each} {#if $documents.length != 0}
From fe997abc6d4908140d3ceba78ebc251b83dce6ab Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Mon, 8 Jan 2024 01:32:55 -0800 Subject: [PATCH 5/9] feat: transform filename to name --- src/lib/utils/index.ts | 13 +++++ src/routes/(app)/documents/+page.svelte | 65 +++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/src/lib/utils/index.ts b/src/lib/utils/index.ts index 75cd073e..129a1d12 100644 --- a/src/lib/utils/index.ts +++ b/src/lib/utils/index.ts @@ -146,6 +146,19 @@ export const removeFirstHashWord = (inputString) => { return resultString; }; +export const transformFileName = (fileName) => { + // Convert to lowercase + const lowerCaseFileName = fileName.toLowerCase(); + + // Remove special characters using regular expression + const sanitizedFileName = lowerCaseFileName.replace(/[^\w\s]/g, ''); + + // Replace spaces with dashes + const finalFileName = sanitizedFileName.replace(/\s+/g, '-'); + + return finalFileName; +}; + export const calculateSHA256 = async (file) => { // Create a FileReader to read the file asynchronously const reader = new FileReader(); diff --git a/src/routes/(app)/documents/+page.svelte b/src/routes/(app)/documents/+page.svelte index 747adec8..f0ad1a54 100644 --- a/src/routes/(app)/documents/+page.svelte +++ b/src/routes/(app)/documents/+page.svelte @@ -9,6 +9,7 @@ import { SUPPORTED_FILE_TYPE } from '$lib/constants'; import { uploadDocToVectorDB } from '$lib/apis/rag'; + import { transformFileName } from '$lib/utils'; let importFiles = ''; @@ -30,7 +31,7 @@ localStorage.token, res.collection_name, res.filename, - res.filename, + transformFileName(res.filename), res.filename ); await documents.set(await getDocs(localStorage.token)); @@ -165,14 +166,70 @@
- -
+
+
+ {#if doc} + + + + + {:else} + + {/if} +
+
#{doc.name} ({doc.filename})
{doc.title}
- +
Date: Mon, 8 Jan 2024 01:49:20 -0800 Subject: [PATCH 6/9] feat: documents backend integration --- backend/apps/web/models/documents.py | 5 +- backend/apps/web/routers/documents.py | 4 +- backend/constants.py | 1 + src/lib/apis/documents/index.ts | 2 +- .../components/documents/EditDocModal.svelte | 151 ++++++++++++++++++ src/routes/(app)/documents/+page.svelte | 20 ++- 6 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 src/lib/components/documents/EditDocModal.svelte diff --git a/backend/apps/web/models/documents.py b/backend/apps/web/models/documents.py index ccd76194..0196c38b 100644 --- a/backend/apps/web/models/documents.py +++ b/backend/apps/web/models/documents.py @@ -105,9 +105,10 @@ class DocumentsTable: ).where(Document.name == name) query.execute() - doc = Document.get(Document.name == name) + doc = Document.get(Document.name == form_data.name) return DocumentModel(**model_to_dict(doc)) - except: + except Exception as e: + print(e) return None def delete_doc_by_name(self, name: str) -> bool: diff --git a/backend/apps/web/routers/documents.py b/backend/apps/web/routers/documents.py index 42463f51..9cc0dd5d 100644 --- a/backend/apps/web/routers/documents.py +++ b/backend/apps/web/routers/documents.py @@ -97,8 +97,8 @@ async def update_doc_by_name( return doc else: raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail=ERROR_MESSAGES.ACCESS_PROHIBITED, + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.NAME_TAG_TAKEN, ) diff --git a/backend/constants.py b/backend/constants.py index 0f7a46a0..b923ec8b 100644 --- a/backend/constants.py +++ b/backend/constants.py @@ -18,6 +18,7 @@ class ERROR_MESSAGES(str, Enum): "Uh-oh! This username is already registered. Please choose another username." ) COMMAND_TAKEN = "Uh-oh! This command is already registered. Please choose another command string." + NAME_TAG_TAKEN = "Uh-oh! This name tag is already registered. Please choose another name tag string." INVALID_TOKEN = ( "Your session has expired or the token is invalid. Please sign in again." ) diff --git a/src/lib/apis/documents/index.ts b/src/lib/apis/documents/index.ts index 37bdde04..fb208ea4 100644 --- a/src/lib/apis/documents/index.ts +++ b/src/lib/apis/documents/index.ts @@ -111,7 +111,7 @@ type DocUpdateForm = { export const updateDocByName = async (token: string, name: string, form: DocUpdateForm) => { let error = null; - const res = await fetch(`${WEBUI_API_BASE_URL}/prompts/name/${name}/update`, { + const res = await fetch(`${WEBUI_API_BASE_URL}/documents/name/${name}/update`, { method: 'POST', headers: { Accept: 'application/json', diff --git a/src/lib/components/documents/EditDocModal.svelte b/src/lib/components/documents/EditDocModal.svelte new file mode 100644 index 00000000..36be3795 --- /dev/null +++ b/src/lib/components/documents/EditDocModal.svelte @@ -0,0 +1,151 @@ + + + +
+
+
Edit Doc
+ +
+
+ +
+
+
{ + submitHandler(); + }} + > +
+
+
Name Tag
+ +
+
+ # +
+ +
+ + +
+ +
+
Title
+ +
+ +
+
+
+ +
+ +
+
+
+
+
+ + + diff --git a/src/routes/(app)/documents/+page.svelte b/src/routes/(app)/documents/+page.svelte index f0ad1a54..fa290999 100644 --- a/src/routes/(app)/documents/+page.svelte +++ b/src/routes/(app)/documents/+page.svelte @@ -11,11 +11,16 @@ import { uploadDocToVectorDB } from '$lib/apis/rag'; import { transformFileName } from '$lib/utils'; + import EditDocModal from '$lib/components/documents/EditDocModal.svelte'; + let importFiles = ''; let inputFiles = ''; let query = ''; + let showEditDocModal = false; + let selectedDoc; + let dragged = false; const deleteDoc = async (name) => { @@ -70,6 +75,10 @@ }; +{#key selectedDoc} + +{/key} +
@@ -156,7 +165,7 @@
Add Files
- Drop any files here to add to the conversation + Drop any files here to add to my documents
@@ -232,10 +241,13 @@
- { + showEditDocModal = !showEditDocModal; + selectedDoc = doc; + }} > - +