Merge branch 'main' into patch-1

This commit is contained in:
Timothy Jaeryang Baek 2024-01-29 16:24:16 -08:00 committed by GitHub
commit 09534dad6e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 761 additions and 340 deletions

View file

@ -5,7 +5,8 @@ FROM node:alpine as build
WORKDIR /app WORKDIR /app
# wget embedding model weight from alpine (does not exist from slim-buster) # wget embedding model weight from alpine (does not exist from slim-buster)
RUN wget "https://chroma-onnx-models.s3.amazonaws.com/all-MiniLM-L6-v2/onnx.tar.gz" RUN wget "https://chroma-onnx-models.s3.amazonaws.com/all-MiniLM-L6-v2/onnx.tar.gz" -O - | \
tar -xzf - -C /app
COPY package.json package-lock.json ./ COPY package.json package-lock.json ./
RUN npm ci RUN npm ci
@ -34,20 +35,17 @@ COPY ./backend/requirements.txt ./requirements.txt
RUN pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir RUN pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir
RUN pip3 install -r requirements.txt --no-cache-dir RUN pip3 install -r requirements.txt --no-cache-dir
# Install pandoc # Install pandoc and netcat
# RUN python -c "import pypandoc; pypandoc.download_pandoc()" # RUN python -c "import pypandoc; pypandoc.download_pandoc()"
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y pandoc \ && apt-get install -y pandoc netcat-openbsd \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# RUN python -c "from sentence_transformers import SentenceTransformer; model = SentenceTransformer('all-MiniLM-L6-v2')" # RUN python -c "from sentence_transformers import SentenceTransformer; model = SentenceTransformer('all-MiniLM-L6-v2')"
# copy embedding weight from build # copy embedding weight from build
RUN mkdir -p /root/.cache/chroma/onnx_models/all-MiniLM-L6-v2 RUN mkdir -p /root/.cache/chroma/onnx_models/all-MiniLM-L6-v2
COPY --from=build /app/onnx.tar.gz /root/.cache/chroma/onnx_models/all-MiniLM-L6-v2 COPY --from=build /app/onnx /root/.cache/chroma/onnx_models/all-MiniLM-L6-v2/onnx
RUN cd /root/.cache/chroma/onnx_models/all-MiniLM-L6-v2 &&\
tar -xzf onnx.tar.gz
# copy built frontend files # copy built frontend files
COPY --from=build /app/build /app/build COPY --from=build /app/build /app/build

View file

@ -24,6 +24,7 @@ from langchain_community.document_loaders import (
UnstructuredMarkdownLoader, UnstructuredMarkdownLoader,
UnstructuredXMLLoader, UnstructuredXMLLoader,
UnstructuredRSTLoader, UnstructuredRSTLoader,
UnstructuredExcelLoader,
) )
from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma from langchain_community.vectorstores import Chroma
@ -36,7 +37,7 @@ from typing import Optional
import uuid import uuid
import time import time
from utils.misc import calculate_sha256 from utils.misc import calculate_sha256, calculate_sha256_string
from utils.utils import get_current_user from utils.utils import get_current_user
from config import UPLOAD_DIR, EMBED_MODEL, CHROMA_CLIENT, CHUNK_SIZE, CHUNK_OVERLAP from config import UPLOAD_DIR, EMBED_MODEL, CHROMA_CLIENT, CHUNK_SIZE, CHUNK_OVERLAP
from constants import ERROR_MESSAGES from constants import ERROR_MESSAGES
@ -123,10 +124,15 @@ def store_web(form_data: StoreWebForm, user=Depends(get_current_user)):
try: try:
loader = WebBaseLoader(form_data.url) loader = WebBaseLoader(form_data.url)
data = loader.load() data = loader.load()
store_data_in_vector_db(data, form_data.collection_name)
collection_name = form_data.collection_name
if collection_name == "":
collection_name = calculate_sha256_string(form_data.url)[:63]
store_data_in_vector_db(data, collection_name)
return { return {
"status": True, "status": True,
"collection_name": form_data.collection_name, "collection_name": collection_name,
"filename": form_data.url, "filename": form_data.url,
} }
except Exception as e: except Exception as e:
@ -137,6 +143,87 @@ def store_web(form_data: StoreWebForm, user=Depends(get_current_user)):
) )
def get_loader(file, file_path):
file_ext = file.filename.split(".")[-1].lower()
known_type = True
known_source_ext = [
"go",
"py",
"java",
"sh",
"bat",
"ps1",
"cmd",
"js",
"ts",
"css",
"cpp",
"hpp",
"h",
"c",
"cs",
"sql",
"log",
"ini",
"pl",
"pm",
"r",
"dart",
"dockerfile",
"env",
"php",
"hs",
"hsc",
"lua",
"nginxconf",
"conf",
"m",
"mm",
"plsql",
"perl",
"rb",
"rs",
"db2",
"scala",
"bash",
"swift",
"vue",
"svelte",
]
if file_ext == "pdf":
loader = PyPDFLoader(file_path)
elif file_ext == "csv":
loader = CSVLoader(file_path)
elif file_ext == "rst":
loader = UnstructuredRSTLoader(file_path, mode="elements")
elif file_ext == "xml":
loader = UnstructuredXMLLoader(file_path)
elif file_ext == "md":
loader = UnstructuredMarkdownLoader(file_path)
elif file.content_type == "application/epub+zip":
loader = UnstructuredEPubLoader(file_path)
elif (
file.content_type
== "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
or file_ext in ["doc", "docx"]
):
loader = Docx2txtLoader(file_path)
elif file.content_type in [
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
] or file_ext in ["xls", "xlsx"]:
loader = UnstructuredExcelLoader(file_path)
elif file_ext in known_source_ext or file.content_type.find("text/") >= 0:
loader = TextLoader(file_path)
else:
loader = TextLoader(file_path)
known_type = False
return loader, known_type
@app.post("/doc") @app.post("/doc")
def store_doc( def store_doc(
collection_name: Optional[str] = Form(None), collection_name: Optional[str] = Form(None),
@ -146,21 +233,6 @@ def store_doc(
# "https://www.gutenberg.org/files/1727/1727-h/1727-h.htm" # "https://www.gutenberg.org/files/1727/1727-h/1727-h.htm"
print(file.content_type) print(file.content_type)
text_xml=["xml"]
octet_markdown=["md"]
known_source_ext=[
"go", "py", "java", "sh", "bat", "ps1", "cmd", "js", "ts",
"css", "cpp", "hpp","h", "c", "cs", "sql", "log", "ini",
"pl", "pm", "r", "dart", "dockerfile", "env", "php", "hs",
"hsc", "lua", "nginxconf", "conf", "m", "mm", "plsql", "perl",
"rb", "rs", "db2", "scala", "bash", "swift", "vue", "svelte"
]
docx_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document"
known_doc_ext=["doc","docx"]
file_ext=file.filename.split(".")[-1].lower()
known_type=True
try: try:
filename = file.filename filename = file.filename
file_path = f"{UPLOAD_DIR}/{filename}" file_path = f"{UPLOAD_DIR}/{filename}"
@ -174,27 +246,7 @@ def store_doc(
collection_name = calculate_sha256(f)[:63] collection_name = calculate_sha256(f)[:63]
f.close() f.close()
if file_ext=="pdf": loader, known_type = get_loader(file, file_path)
loader = PyPDFLoader(file_path)
elif (file.content_type ==docx_type or file_ext in known_doc_ext):
loader = Docx2txtLoader(file_path)
elif file_ext=="csv":
loader = CSVLoader(file_path)
elif file_ext=="rst":
loader = UnstructuredRSTLoader(file_path, mode="elements")
elif file_ext in text_xml:
loader=UnstructuredXMLLoader(file_path)
elif file_ext in known_source_ext or file.content_type.find("text/")>=0:
loader = TextLoader(file_path)
elif file_ext in octet_markdown:
loader = UnstructuredMarkdownLoader(file_path)
elif file.content_type == "application/epub+zip":
loader = UnstructuredEPubLoader(file_path)
else:
loader = TextLoader(file_path)
known_type=False
data = loader.load() data = loader.load()
result = store_data_in_vector_db(data, collection_name) result = store_data_in_vector_db(data, collection_name)
@ -203,7 +255,7 @@ def store_doc(
"status": True, "status": True,
"collection_name": collection_name, "collection_name": collection_name,
"filename": filename, "filename": filename,
"known_type":known_type, "known_type": known_type,
} }
else: else:
raise HTTPException( raise HTTPException(

View file

@ -1,4 +1,6 @@
from peewee import * from peewee import *
from config import DATA_DIR
DB = SqliteDatabase("./data/ollama.db")
DB = SqliteDatabase(f"{DATA_DIR}/ollama.db")
DB.connect() DB.connect()

View file

@ -63,6 +63,15 @@ class SigninForm(BaseModel):
password: str password: str
class ProfileImageUrlForm(BaseModel):
profile_image_url: str
class UpdateProfileForm(BaseModel):
profile_image_url: str
name: str
class UpdatePasswordForm(BaseModel): class UpdatePasswordForm(BaseModel):
password: str password: str
new_password: str new_password: str

View file

@ -65,7 +65,7 @@ class UsersTable:
"name": name, "name": name,
"email": email, "email": email,
"role": role, "role": role,
"profile_image_url": get_gravatar_url(email), "profile_image_url": "/user.png",
"timestamp": int(time.time()), "timestamp": int(time.time()),
} }
) )
@ -108,6 +108,20 @@ class UsersTable:
except: except:
return None return None
def update_user_profile_image_url_by_id(
self, id: str, profile_image_url: str
) -> Optional[UserModel]:
try:
query = User.update(profile_image_url=profile_image_url).where(
User.id == id
)
query.execute()
user = User.get(User.id == id)
return UserModel(**model_to_dict(user))
except:
return None
def update_user_by_id(self, id: str, updated: dict) -> Optional[UserModel]: def update_user_by_id(self, id: str, updated: dict) -> Optional[UserModel]:
try: try:
query = User.update(**updated).where(User.id == id) query = User.update(**updated).where(User.id == id)

View file

@ -11,6 +11,7 @@ import uuid
from apps.web.models.auths import ( from apps.web.models.auths import (
SigninForm, SigninForm,
SignupForm, SignupForm,
UpdateProfileForm,
UpdatePasswordForm, UpdatePasswordForm,
UserResponse, UserResponse,
SigninResponse, SigninResponse,
@ -40,14 +41,37 @@ async def get_session_user(user=Depends(get_current_user)):
} }
############################
# Update Profile
############################
@router.post("/update/profile", response_model=UserResponse)
async def update_profile(
form_data: UpdateProfileForm, session_user=Depends(get_current_user)
):
if session_user:
user = Users.update_user_by_id(
session_user.id,
{"profile_image_url": form_data.profile_image_url, "name": form_data.name},
)
if user:
return user
else:
raise HTTPException(400, detail=ERROR_MESSAGES.DEFAULT())
else:
raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED)
############################ ############################
# Update Password # Update Password
############################ ############################
@router.post("/update/password", response_model=bool) @router.post("/update/password", response_model=bool)
async def update_password(form_data: UpdatePasswordForm, async def update_password(
session_user=Depends(get_current_user)): form_data: UpdatePasswordForm, session_user=Depends(get_current_user)
):
if session_user: if session_user:
user = Auths.authenticate_user(session_user.email, form_data.password) user = Auths.authenticate_user(session_user.email, form_data.password)
@ -103,8 +127,9 @@ async def signup(request: Request, form_data: SignupForm):
try: try:
role = "admin" if Users.get_num_users() == 0 else "pending" role = "admin" if Users.get_num_users() == 0 else "pending"
hashed = get_password_hash(form_data.password) hashed = get_password_hash(form_data.password)
user = Auths.insert_new_auth(form_data.email.lower(), user = Auths.insert_new_auth(
hashed, form_data.name, role) form_data.email.lower(), hashed, form_data.name, role
)
if user: if user:
token = create_token(data={"email": user.email}) token = create_token(data={"email": user.email})
@ -120,11 +145,10 @@ async def signup(request: Request, form_data: SignupForm):
"profile_image_url": user.profile_image_url, "profile_image_url": user.profile_image_url,
} }
else: else:
raise HTTPException( raise HTTPException(500, detail=ERROR_MESSAGES.CREATE_USER_ERROR)
500, detail=ERROR_MESSAGES.CREATE_USER_ERROR)
except Exception as err: except Exception as err:
raise HTTPException(500, raise HTTPException(500, detail=ERROR_MESSAGES.DEFAULT(err))
detail=ERROR_MESSAGES.DEFAULT(err))
############################ ############################
# ToggleSignUp # ToggleSignUp

View file

@ -9,9 +9,9 @@ import os
import aiohttp import aiohttp
import json import json
from utils.misc import calculate_sha256 from utils.misc import calculate_sha256, get_gravatar_url
from config import OLLAMA_API_BASE_URL from config import OLLAMA_API_BASE_URL, DATA_DIR, UPLOAD_DIR
from constants import ERROR_MESSAGES from constants import ERROR_MESSAGES
@ -96,8 +96,7 @@ async def download(
file_name = parse_huggingface_url(url) file_name = parse_huggingface_url(url)
if file_name: if file_name:
os.makedirs("./uploads", exist_ok=True) file_path = f"{UPLOAD_DIR}/{file_name}"
file_path = os.path.join("./uploads", f"{file_name}")
return StreamingResponse( return StreamingResponse(
download_file_stream(url, file_path, file_name), download_file_stream(url, file_path, file_name),
@ -109,8 +108,7 @@ async def download(
@router.post("/upload") @router.post("/upload")
def upload(file: UploadFile = File(...)): def upload(file: UploadFile = File(...)):
os.makedirs("./data/uploads", exist_ok=True) file_path = f"{UPLOAD_DIR}/{file.filename}"
file_path = os.path.join("./data/uploads", file.filename)
# Save file in chunks # Save file in chunks
with open(file_path, "wb+") as f: with open(file_path, "wb+") as f:
@ -167,3 +165,10 @@ def upload(file: UploadFile = File(...)):
yield f"data: {json.dumps(res)}\n\n" yield f"data: {json.dumps(res)}\n\n"
return StreamingResponse(file_process_stream(), media_type="text/event-stream") return StreamingResponse(file_process_stream(), media_type="text/event-stream")
@router.get("/gravatar")
async def get_gravatar(
email: str,
):
return get_gravatar_url(email)

View file

@ -1,29 +1,17 @@
from dotenv import load_dotenv, find_dotenv
import os import os
import chromadb import chromadb
from chromadb import Settings from chromadb import Settings
from secrets import token_bytes from secrets import token_bytes
from base64 import b64encode from base64 import b64encode
from constants import ERROR_MESSAGES from constants import ERROR_MESSAGES
from pathlib import Path from pathlib import Path
load_dotenv(find_dotenv("../.env")) try:
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv("../.env"))
#################################### except ImportError:
# File Upload print("dotenv not installed, skipping...")
####################################
UPLOAD_DIR = "./data/uploads"
Path(UPLOAD_DIR).mkdir(parents=True, exist_ok=True)
#################################### ####################################
@ -32,6 +20,21 @@ Path(UPLOAD_DIR).mkdir(parents=True, exist_ok=True)
ENV = os.environ.get("ENV", "dev") ENV = os.environ.get("ENV", "dev")
####################################
# DATA/FRONTEND BUILD DIR
####################################
DATA_DIR = str(Path(os.getenv("DATA_DIR", "./data")).resolve())
FRONTEND_BUILD_DIR = str(Path(os.getenv("FRONTEND_BUILD_DIR", "../build")))
####################################
# File Upload DIR
####################################
UPLOAD_DIR = f"{DATA_DIR}/uploads"
Path(UPLOAD_DIR).mkdir(parents=True, exist_ok=True)
#################################### ####################################
# OLLAMA_API_BASE_URL # OLLAMA_API_BASE_URL
#################################### ####################################
@ -107,7 +110,7 @@ if WEBUI_AUTH and WEBUI_JWT_SECRET_KEY == "":
# RAG # RAG
#################################### ####################################
CHROMA_DATA_PATH = "./data/vector_db" CHROMA_DATA_PATH = f"{DATA_DIR}/vector_db"
EMBED_MODEL = "all-MiniLM-L6-v2" EMBED_MODEL = "all-MiniLM-L6-v2"
CHROMA_CLIENT = chromadb.PersistentClient( CHROMA_CLIENT = chromadb.PersistentClient(
path=CHROMA_DATA_PATH, settings=Settings(allow_reset=True) path=CHROMA_DATA_PATH, settings=Settings(allow_reset=True)

View file

@ -14,7 +14,7 @@ from apps.openai.main import app as openai_app
from apps.web.main import app as webui_app from apps.web.main import app as webui_app
from apps.rag.main import app as rag_app from apps.rag.main import app as rag_app
from config import ENV from config import ENV, FRONTEND_BUILD_DIR
class SPAStaticFiles(StaticFiles): class SPAStaticFiles(StaticFiles):
@ -58,4 +58,8 @@ app.mount("/openai/api", openai_app)
app.mount("/rag/api/v1", rag_app) app.mount("/rag/api/v1", rag_app)
app.mount("/", SPAStaticFiles(directory="../build", html=True), name="spa-static-files") app.mount(
"/",
SPAStaticFiles(directory=FRONTEND_BUILD_DIR, html=True),
name="spa-static-files",
)

View file

@ -25,6 +25,10 @@ docx2txt
unstructured unstructured
markdown markdown
pypandoc pypandoc
pandas
openpyxl
pyxlsb
xlrd
PyJWT PyJWT
pyjwt[crypto] pyjwt[crypto]

View file

@ -4,4 +4,4 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
cd "$SCRIPT_DIR" || exit cd "$SCRIPT_DIR" || exit
PORT="${PORT:-8080}" PORT="${PORT:-8080}"
uvicorn main:app --host 0.0.0.0 --port "$PORT" --forwarded-allow-ips '*' exec uvicorn main:app --host 0.0.0.0 --port "$PORT" --forwarded-allow-ips '*'

View file

@ -24,6 +24,16 @@ def calculate_sha256(file):
return sha256.hexdigest() return sha256.hexdigest()
def calculate_sha256_string(string):
# Create a new SHA-256 hash object
sha256_hash = hashlib.sha256()
# Update the hash object with the bytes of the input string
sha256_hash.update(string.encode("utf-8"))
# Get the hexadecimal representation of the hash
hashed_string = sha256_hash.hexdigest()
return hashed_string
def validate_email_format(email: str) -> bool: def validate_email_format(email: str) -> bool:
if not re.match(r"[^@]+@[^@]+\.[^@]+", email): if not re.match(r"[^@]+@[^@]+\.[^@]+", email):
return False return False

20
docs/SECURITY.md Normal file
View file

@ -0,0 +1,20 @@
# Security Policy
Our primary goal is to ensure the protection and confidentiality of sensitive data stored by users on ollama-webui.
## Supported Versions
| Version | Supported |
| ------- | ------------------ |
| main | :white_check_mark: |
| others | :x: |
## Reporting a Vulnerability
If you discover a security issue within our system, please notify us immediately via a pull request or contact us on discord.
## Product Security
We regularly audit our internal processes and system's architecture for vulnerabilities using a combination of automated and manual testing techniques.
We are planning on implementing SAST and SCA scans in our project soon.

View file

@ -11,8 +11,8 @@ TICK='\u2713'
# Detect GPU driver # Detect GPU driver
get_gpu_driver() { get_gpu_driver() {
# Detect NVIDIA GPUs # Detect NVIDIA GPUs using lspci or nvidia-smi
if lspci | grep -i nvidia >/dev/null; then if lspci | grep -i nvidia >/dev/null || nvidia-smi >/dev/null 2>&1; then
echo "nvidia" echo "nvidia"
return return
fi fi
@ -181,6 +181,9 @@ else
DEFAULT_COMPOSE_COMMAND+=" -f docker-compose.data.yaml" DEFAULT_COMPOSE_COMMAND+=" -f docker-compose.data.yaml"
export OLLAMA_DATA_DIR=$data_dir # Set OLLAMA_DATA_DIR environment variable export OLLAMA_DATA_DIR=$data_dir # Set OLLAMA_DATA_DIR environment variable
fi fi
if [[ -n $webui_port ]]; then
export OLLAMA_WEBUI_PORT=$webui_port # Set OLLAMA_WEBUI_PORT environment variable
fi
DEFAULT_COMPOSE_COMMAND+=" up -d" DEFAULT_COMPOSE_COMMAND+=" up -d"
DEFAULT_COMPOSE_COMMAND+=" --remove-orphans" DEFAULT_COMPOSE_COMMAND+=" --remove-orphans"
DEFAULT_COMPOSE_COMMAND+=" --force-recreate" DEFAULT_COMPOSE_COMMAND+=" --force-recreate"

View file

@ -10,10 +10,10 @@ docker pull ollama/ollama:latest
docker_args="-d -v ollama:/root/.ollama -p $host_port:$container_port --name ollama ollama/ollama" docker_args="-d -v ollama:/root/.ollama -p $host_port:$container_port --name ollama ollama/ollama"
if [ "$use_gpu" == "y" ]; then if [ "$use_gpu" = "y" ]; then
docker_args+=" --gpus=all" docker_args="--gpus=all $docker_args"
fi fi
docker run "$docker_args" docker run $docker_args
docker image prune -f docker image prune -f

View file

@ -89,6 +89,37 @@ export const userSignUp = async (name: string, email: string, password: string)
return res; return res;
}; };
export const updateUserProfile = async (token: string, name: string, profileImageUrl: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/auths/update/profile`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
...(token && { authorization: `Bearer ${token}` })
},
body: JSON.stringify({
name: name,
profile_image_url: profileImageUrl
})
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
console.log(err);
error = err.detail;
return null;
});
if (error) {
throw error;
}
return res;
};
export const updateUserPassword = async (token: string, password: string, newPassword: string) => { export const updateUserPassword = async (token: string, password: string, newPassword: string) => {
let error = null; let error = null;

View file

@ -0,0 +1,23 @@
import { WEBUI_API_BASE_URL } from '$lib/constants';
export const getGravatarUrl = async (email: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/utils/gravatar?email=${email}`, {
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
console.log(err);
error = err;
return null;
});
return res;
};

View file

@ -6,7 +6,7 @@
import Prompts from './MessageInput/PromptCommands.svelte'; import Prompts from './MessageInput/PromptCommands.svelte';
import Suggestions from './MessageInput/Suggestions.svelte'; import Suggestions from './MessageInput/Suggestions.svelte';
import { uploadDocToVectorDB } from '$lib/apis/rag'; import { uploadDocToVectorDB, uploadWebToVectorDB } from '$lib/apis/rag';
import AddFilesPlaceholder from '../AddFilesPlaceholder.svelte'; import AddFilesPlaceholder from '../AddFilesPlaceholder.svelte';
import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS } from '$lib/constants'; import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS } from '$lib/constants';
import Documents from './MessageInput/Documents.svelte'; import Documents from './MessageInput/Documents.svelte';
@ -137,6 +137,33 @@
} }
}; };
const uploadWeb = async (url) => {
console.log(url);
const doc = {
type: 'doc',
name: url,
collection_name: '',
upload_status: false,
error: ''
};
try {
files = [...files, doc];
const res = await uploadWebToVectorDB(localStorage.token, '', url);
if (res) {
doc.upload_status = true;
doc.collection_name = res.collection_name;
files = files;
}
} catch (e) {
// Remove the failed doc from the files array
files = files.filter((f) => f.name !== url);
toast.error(e);
}
};
onMount(() => { onMount(() => {
const dropZone = document.querySelector('body'); const dropZone = document.querySelector('body');
@ -258,6 +285,10 @@
<Documents <Documents
bind:this={documentsElement} bind:this={documentsElement}
bind:prompt bind:prompt
on:url={(e) => {
console.log(e);
uploadWeb(e.detail);
}}
on:select={(e) => { on:select={(e) => {
console.log(e); console.log(e);
files = [ files = [

View file

@ -2,8 +2,9 @@
import { createEventDispatcher } from 'svelte'; import { createEventDispatcher } from 'svelte';
import { documents } from '$lib/stores'; import { documents } from '$lib/stores';
import { removeFirstHashWord } from '$lib/utils'; import { removeFirstHashWord, isValidHttpUrl } from '$lib/utils';
import { tick } from 'svelte'; import { tick } from 'svelte';
import toast from 'svelte-french-toast';
export let prompt = ''; export let prompt = '';
@ -37,9 +38,20 @@
chatInputElement?.focus(); chatInputElement?.focus();
await tick(); await tick();
}; };
const confirmSelectWeb = async (url) => {
dispatch('url', url);
prompt = removeFirstHashWord(prompt);
const chatInputElement = document.getElementById('chat-textarea');
await tick();
chatInputElement?.focus();
await tick();
};
</script> </script>
{#if filteredDocs.length > 0} {#if filteredDocs.length > 0 || prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
<div class="md:px-2 mb-3 text-left w-full"> <div class="md:px-2 mb-3 text-left w-full">
<div class="flex w-full rounded-lg border border-gray-100 dark:border-gray-700"> <div class="flex w-full rounded-lg border border-gray-100 dark:border-gray-700">
<div class=" bg-gray-100 dark:bg-gray-700 w-10 rounded-l-lg text-center"> <div class=" bg-gray-100 dark:bg-gray-700 w-10 rounded-l-lg text-center">
@ -55,6 +67,7 @@
: ''}" : ''}"
type="button" type="button"
on:click={() => { on:click={() => {
console.log(doc);
confirmSelect(doc); confirmSelect(doc);
}} }}
on:mousemove={() => { on:mousemove={() => {
@ -71,6 +84,29 @@
</div> </div>
</button> </button>
{/each} {/each}
{#if prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
<button
class="px-3 py-1.5 rounded-lg w-full text-left bg-gray-100 selected-command-option-button"
type="button"
on:click={() => {
const url = prompt.split(' ')?.at(0)?.substring(1);
if (isValidHttpUrl(url)) {
confirmSelectWeb(url);
} else {
toast.error(
'Oops! Looks like the URL is invalid. Please double-check and try again.'
);
}
}}
>
<div class=" font-medium text-black line-clamp-1">
{prompt.split(' ')?.at(0)?.substring(1)}
</div>
<div class=" text-xs text-gray-600 line-clamp-1">Web</div>
</button>
{/if}
</div> </div>
</div> </div>
</div> </div>

View file

@ -1,6 +1,7 @@
<script lang="ts"> <script lang="ts">
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import { marked } from 'marked'; import { marked } from 'marked';
import { settings, voices } from '$lib/stores';
import tippy from 'tippy.js'; import tippy from 'tippy.js';
import auto_render from 'katex/dist/contrib/auto-render.mjs'; import auto_render from 'katex/dist/contrib/auto-render.mjs';
import 'katex/dist/katex.min.css'; import 'katex/dist/katex.min.css';
@ -116,6 +117,8 @@
} else { } else {
speaking = true; speaking = true;
const speak = new SpeechSynthesisUtterance(message.content); const speak = new SpeechSynthesisUtterance(message.content);
const voice = $voices?.filter((v) => v.name === $settings?.speakVoice)?.at(0) ?? undefined;
speak.voice = voice;
speechSynthesis.speak(speak); speechSynthesis.speak(speak);
} }
}; };

View file

@ -0,0 +1,179 @@
<script lang="ts">
import toast from 'svelte-french-toast';
import { onMount } from 'svelte';
import { user } from '$lib/stores';
import { updateUserProfile } from '$lib/apis/auths';
import UpdatePassword from './Account/UpdatePassword.svelte';
import { getGravatarUrl } from '$lib/apis/utils';
export let saveHandler: Function;
let profileImageUrl = '';
let name = '';
const submitHandler = async () => {
const updatedUser = await updateUserProfile(localStorage.token, name, profileImageUrl).catch(
(error) => {
toast.error(error);
}
);
if (updatedUser) {
await user.set(updatedUser);
return true;
}
return false;
};
onMount(() => {
name = $user.name;
profileImageUrl = $user.profile_image_url;
});
</script>
<div class="flex flex-col h-full justify-between text-sm">
<div class=" space-y-3 pr-1.5 overflow-y-scroll max-h-80">
<input
id="profile-image-input"
type="file"
hidden
accept="image/*"
on:change={(e) => {
const files = e?.target?.files ?? [];
let reader = new FileReader();
reader.onload = (event) => {
let originalImageUrl = `${event.target.result}`;
const img = new Image();
img.src = originalImageUrl;
img.onload = function () {
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d');
// Calculate the aspect ratio of the image
const aspectRatio = img.width / img.height;
// Calculate the new width and height to fit within 100x100
let newWidth, newHeight;
if (aspectRatio > 1) {
newWidth = 100 * aspectRatio;
newHeight = 100;
} else {
newWidth = 100;
newHeight = 100 / aspectRatio;
}
// Set the canvas size
canvas.width = 100;
canvas.height = 100;
// Calculate the position to center the image
const offsetX = (100 - newWidth) / 2;
const offsetY = (100 - newHeight) / 2;
// Draw the image on the canvas
ctx.drawImage(img, offsetX, offsetY, newWidth, newHeight);
// Get the base64 representation of the compressed image
const compressedSrc = canvas.toDataURL('image/jpeg');
// Display the compressed image
profileImageUrl = compressedSrc;
e.target.files = null;
};
};
if (
files.length > 0 &&
['image/gif', 'image/jpeg', 'image/png'].includes(files[0]['type'])
) {
reader.readAsDataURL(files[0]);
}
}}
/>
<div class=" mb-2.5 text-sm font-medium">Profile</div>
<div class="flex space-x-5">
<div class="flex flex-col">
<div class="self-center">
<button
class="relative rounded-full dark:bg-gray-700"
type="button"
on:click={() => {
document.getElementById('profile-image-input')?.click();
}}
>
<img
src={profileImageUrl !== '' ? profileImageUrl : '/user.png'}
alt="profile"
class=" rounded-full w-16 h-16 object-cover"
/>
<div
class="absolute flex justify-center rounded-full bottom-0 left-0 right-0 top-0 h-full w-full overflow-hidden bg-gray-700 bg-fixed opacity-0 transition duration-300 ease-in-out hover:opacity-50"
>
<div class="my-auto text-gray-100">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="w-5 h-5"
>
<path
d="m2.695 14.762-1.262 3.155a.5.5 0 0 0 .65.65l3.155-1.262a4 4 0 0 0 1.343-.886L17.5 5.501a2.121 2.121 0 0 0-3-3L3.58 13.419a4 4 0 0 0-.885 1.343Z"
/>
</svg>
</div>
</div>
</button>
</div>
<button
class=" text-xs text-gray-600"
on:click={async () => {
const url = await getGravatarUrl($user.email);
profileImageUrl = url;
}}>Use Gravatar</button
>
</div>
<div class="flex-1">
<div class="flex flex-col w-full">
<div class=" mb-1 text-xs text-gray-500">Name</div>
<div class="flex-1">
<input
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
type="text"
bind:value={name}
required
/>
</div>
</div>
</div>
</div>
<hr class=" dark:border-gray-700 my-4" />
<UpdatePassword />
</div>
<div class="flex justify-end pt-3 text-sm font-medium">
<button
class=" px-4 py-2 bg-emerald-600 hover:bg-emerald-700 text-gray-100 transition rounded"
on:click={async () => {
const res = await submitHandler();
if (res) {
saveHandler();
}
}}
>
Save
</button>
</div>
</div>

View file

@ -0,0 +1,106 @@
<script lang="ts">
import toast from 'svelte-french-toast';
import { updateUserPassword } from '$lib/apis/auths';
let show = false;
let currentPassword = '';
let newPassword = '';
let newPasswordConfirm = '';
const updatePasswordHandler = async () => {
if (newPassword === newPasswordConfirm) {
const res = await updateUserPassword(localStorage.token, currentPassword, newPassword).catch(
(error) => {
toast.error(error);
return null;
}
);
if (res) {
toast.success('Successfully updated.');
}
currentPassword = '';
newPassword = '';
newPasswordConfirm = '';
} else {
toast.error(
`The passwords you entered don't quite match. Please double-check and try again.`
);
newPassword = '';
newPasswordConfirm = '';
}
};
</script>
<form
class="flex flex-col text-sm"
on:submit|preventDefault={() => {
updatePasswordHandler();
}}
>
<div class="flex justify-between mb-2.5 items-center text-sm">
<div class=" font-medium">Change Password</div>
<button
class=" text-xs font-medium text-gray-500"
type="button"
on:click={() => {
show = !show;
}}>{show ? 'Hide' : 'Show'}</button
>
</div>
{#if show}
<div class=" space-y-1.5">
<div class="flex flex-col w-full">
<div class=" mb-1 text-xs text-gray-500">Current Password</div>
<div class="flex-1">
<input
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
type="password"
bind:value={currentPassword}
autocomplete="current-password"
required
/>
</div>
</div>
<div class="flex flex-col w-full">
<div class=" mb-1 text-xs text-gray-500">New Password</div>
<div class="flex-1">
<input
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
type="password"
bind:value={newPassword}
autocomplete="new-password"
required
/>
</div>
</div>
<div class="flex flex-col w-full">
<div class=" mb-1 text-xs text-gray-500">Confirm Password</div>
<div class="flex-1">
<input
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
type="password"
bind:value={newPasswordConfirm}
autocomplete="off"
required
/>
</div>
</div>
</div>
<div class="mt-3 flex justify-end">
<button
class=" px-4 py-2 text-xs bg-gray-800 hover:bg-gray-900 dark:bg-gray-700 dark:hover:bg-gray-800 text-gray-100 transition rounded-md font-medium"
>
Update password
</button>
</div>
{/if}
</form>

View file

@ -20,7 +20,7 @@
import { createNewChat, deleteAllChats, getAllChats, getChatList } from '$lib/apis/chats'; import { createNewChat, deleteAllChats, getAllChats, getChatList } from '$lib/apis/chats';
import { WEB_UI_VERSION, WEBUI_API_BASE_URL } from '$lib/constants'; import { WEB_UI_VERSION, WEBUI_API_BASE_URL } from '$lib/constants';
import { config, models, settings, user, chats } from '$lib/stores'; import { config, models, voices, settings, user, chats } from '$lib/stores';
import { splitStream, getGravatarURL, getImportOrigin, convertOpenAIChats } from '$lib/utils'; import { splitStream, getGravatarURL, getImportOrigin, convertOpenAIChats } from '$lib/utils';
import Advanced from './Settings/Advanced.svelte'; import Advanced from './Settings/Advanced.svelte';
@ -36,6 +36,8 @@
import { resetVectorDB } from '$lib/apis/rag'; import { resetVectorDB } from '$lib/apis/rag';
import { setDefaultPromptSuggestions } from '$lib/apis/configs'; import { setDefaultPromptSuggestions } from '$lib/apis/configs';
import { getBackendConfig } from '$lib/apis'; import { getBackendConfig } from '$lib/apis';
import UpdatePassword from './Settings/Account/UpdatePassword.svelte';
import Account from './Settings/Account.svelte';
export let show = false; export let show = false;
@ -112,6 +114,9 @@
let gravatarEmail = ''; let gravatarEmail = '';
let titleAutoGenerateModel = ''; let titleAutoGenerateModel = '';
// Voice
let speakVoice = '';
// Chats // Chats
let saveChatHistory = true; let saveChatHistory = true;
let importFiles; let importFiles;
@ -123,6 +128,7 @@
let authContent = ''; let authContent = '';
// Account // Account
let profileImageUrl = '';
let currentPassword = ''; let currentPassword = '';
let newPassword = ''; let newPassword = '';
let newPasswordConfirm = ''; let newPasswordConfirm = '';
@ -556,31 +562,6 @@
return models; return models;
}; };
const updatePasswordHandler = async () => {
if (newPassword === newPasswordConfirm) {
const res = await updateUserPassword(localStorage.token, currentPassword, newPassword).catch(
(error) => {
toast.error(error);
return null;
}
);
if (res) {
toast.success('Successfully updated.');
}
currentPassword = '';
newPassword = '';
newPasswordConfirm = '';
} else {
toast.error(
`The passwords you entered don't quite match. Please double-check and try again.`
);
newPassword = '';
newPasswordConfirm = '';
}
};
onMount(async () => { onMount(async () => {
console.log('settings', $user.role === 'admin'); console.log('settings', $user.role === 'admin');
if ($user.role === 'admin') { if ($user.role === 'admin') {
@ -613,15 +594,20 @@
responseAutoCopy = settings.responseAutoCopy ?? false; responseAutoCopy = settings.responseAutoCopy ?? false;
titleAutoGenerateModel = settings.titleAutoGenerateModel ?? ''; titleAutoGenerateModel = settings.titleAutoGenerateModel ?? '';
gravatarEmail = settings.gravatarEmail ?? ''; gravatarEmail = settings.gravatarEmail ?? '';
speakVoice = settings.speakVoice ?? '';
const getVoicesLoop = setInterval(async () => {
const _voices = await speechSynthesis.getVoices();
await voices.set(_voices);
// do your loop
if (_voices.length > 0) {
clearInterval(getVoicesLoop);
}
}, 100);
saveChatHistory = settings.saveChatHistory ?? true; saveChatHistory = settings.saveChatHistory ?? true;
authEnabled = settings.authHeader !== undefined ? true : false;
if (authEnabled) {
authType = settings.authHeader.split(' ')[0];
authContent = settings.authHeader.split(' ')[1];
}
ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => { ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => {
return ''; return '';
}); });
@ -1602,6 +1588,9 @@
<form <form
class="flex flex-col h-full justify-between space-y-3 text-sm" class="flex flex-col h-full justify-between space-y-3 text-sm"
on:submit|preventDefault={() => { on:submit|preventDefault={() => {
saveSettings({
speakVoice: speakVoice !== '' ? speakVoice : undefined
});
show = false; show = false;
}} }}
> >
@ -1683,7 +1672,7 @@
bind:value={titleAutoGenerateModel} bind:value={titleAutoGenerateModel}
placeholder="Select a model" placeholder="Select a model"
> >
<option value="" selected>Default</option> <option value="" selected>Current Model</option>
{#each $models.filter((m) => m.size != null) as model} {#each $models.filter((m) => m.size != null) as model}
<option value={model.name} class="bg-gray-100 dark:bg-gray-700" <option value={model.name} class="bg-gray-100 dark:bg-gray-700"
>{model.name + >{model.name +
@ -1720,7 +1709,31 @@
</div> </div>
</div> </div>
<!-- <hr class=" dark:border-gray-700" /> <hr class=" dark:border-gray-700" />
<div class=" space-y-3">
<div>
<div class=" mb-2.5 text-sm font-medium">Set Default Voice</div>
<div class="flex w-full">
<div class="flex-1">
<select
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
bind:value={speakVoice}
placeholder="Select a voice"
>
<option value="" selected>Default</option>
{#each $voices.filter((v) => v.localService === true) as voice}
<option value={voice.name} class="bg-gray-100 dark:bg-gray-700"
>{voice.name}</option
>
{/each}
</select>
</div>
</div>
</div>
</div>
<!--
<div> <div>
<div class=" mb-2.5 text-sm font-medium"> <div class=" mb-2.5 text-sm font-medium">
Gravatar Email <span class=" text-gray-400 text-sm">(optional)</span> Gravatar Email <span class=" text-gray-400 text-sm">(optional)</span>
@ -1998,184 +2011,12 @@
{/if} {/if}
</div> </div>
</div> </div>
{:else if selectedTab === 'auth'} {:else if selectedTab === 'account'}
<form <Account
class="flex flex-col h-full justify-between space-y-3 text-sm" saveHandler={() => {
on:submit|preventDefault={() => {
console.log('auth save');
saveSettings({
authHeader: authEnabled ? `${authType} ${authContent}` : undefined
});
show = false; show = false;
}} }}
> />
<div class=" space-y-3">
<div>
<div class=" py-1 flex w-full justify-between">
<div class=" self-center text-sm font-medium">Authorization Header</div>
<button
class="p-1 px-3 text-xs flex rounded transition"
type="button"
on:click={() => {
toggleAuthHeader();
}}
>
{#if authEnabled === true}
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="w-4 h-4"
>
<path
fill-rule="evenodd"
d="M12 1.5a5.25 5.25 0 00-5.25 5.25v3a3 3 0 00-3 3v6.75a3 3 0 003 3h10.5a3 3 0 003-3v-6.75a3 3 0 00-3-3v-3c0-2.9-2.35-5.25-5.25-5.25zm3.75 8.25v-3a3.75 3.75 0 10-7.5 0v3h7.5z"
clip-rule="evenodd"
/>
</svg>
<span class="ml-2 self-center"> On </span>
{:else}
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="w-4 h-4"
>
<path
d="M18 1.5c2.9 0 5.25 2.35 5.25 5.25v3.75a.75.75 0 01-1.5 0V6.75a3.75 3.75 0 10-7.5 0v3a3 3 0 013 3v6.75a3 3 0 01-3 3H3.75a3 3 0 01-3-3v-6.75a3 3 0 013-3h9v-3c0-2.9 2.35-5.25 5.25-5.25z"
/>
</svg>
<span class="ml-2 self-center">Off</span>
{/if}
</button>
</div>
</div>
{#if authEnabled}
<hr class=" dark:border-gray-700" />
<div class="mt-2">
<div class=" py-1 flex w-full space-x-2">
<button
class=" py-1 font-semibold flex rounded transition"
on:click={() => {
authType = authType === 'Basic' ? 'Bearer' : 'Basic';
}}
type="button"
>
{#if authType === 'Basic'}
<span class="self-center mr-2">Basic</span>
{:else if authType === 'Bearer'}
<span class="self-center mr-2">Bearer</span>
{/if}
</button>
<div class="flex-1">
<input
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
placeholder="Enter Authorization Header Content"
bind:value={authContent}
/>
</div>
</div>
<div class="mt-2 text-xs text-gray-400 dark:text-gray-500">
Toggle between <span class=" text-gray-500 dark:text-gray-300 font-medium"
>'Basic'</span
>
and <span class=" text-gray-500 dark:text-gray-300 font-medium">'Bearer'</span> by
clicking on the label next to the input.
</div>
</div>
<hr class=" dark:border-gray-700" />
<div>
<div class=" mb-2.5 text-sm font-medium">Preview Authorization Header</div>
<textarea
value={JSON.stringify({
Authorization: `${authType} ${authContent}`
})}
class="w-full rounded p-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none resize-none"
rows="2"
disabled
/>
</div>
{/if}
</div>
<div class="flex justify-end pt-3 text-sm font-medium">
<button
class=" px-4 py-2 bg-emerald-600 hover:bg-emerald-700 text-gray-100 transition rounded"
type="submit"
>
Save
</button>
</div>
</form>
{:else if selectedTab === 'account'}
<form
class="flex flex-col h-full text-sm"
on:submit|preventDefault={() => {
updatePasswordHandler();
}}
>
<div class=" mb-2.5 font-medium">Change Password</div>
<div class=" space-y-1.5">
<div class="flex flex-col w-full">
<div class=" mb-1 text-xs text-gray-500">Current Password</div>
<div class="flex-1">
<input
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
type="password"
bind:value={currentPassword}
autocomplete="current-password"
required
/>
</div>
</div>
<div class="flex flex-col w-full">
<div class=" mb-1 text-xs text-gray-500">New Password</div>
<div class="flex-1">
<input
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
type="password"
bind:value={newPassword}
autocomplete="new-password"
required
/>
</div>
</div>
<div class="flex flex-col w-full">
<div class=" mb-1 text-xs text-gray-500">Confirm Password</div>
<div class="flex-1">
<input
class="w-full rounded py-2 px-4 text-sm dark:text-gray-300 dark:bg-gray-800 outline-none"
type="password"
bind:value={newPasswordConfirm}
autocomplete="off"
required
/>
</div>
</div>
</div>
<div class="mt-3 flex justify-end">
<button
class=" px-4 py-2 text-xs bg-gray-800 hover:bg-gray-900 dark:bg-gray-700 dark:hover:bg-gray-800 text-gray-100 transition rounded-md font-medium"
>
Update password
</button>
</div>
</form>
{:else if selectedTab === 'about'} {:else if selectedTab === 'about'}
<div class="flex flex-col h-full justify-between space-y-3 text-sm mb-6"> <div class="flex flex-col h-full justify-between space-y-3 text-sm mb-6">
<div class=" space-y-3"> <div class=" space-y-3">

View file

@ -321,8 +321,9 @@
return true; return true;
} else { } else {
let title = chat.title.toLowerCase(); let title = chat.title.toLowerCase();
const query = search.toLowerCase();
if (title.includes(search)) { if (title.includes(query)) {
return true; return true;
} else { } else {
return false; return false;

View file

@ -31,7 +31,7 @@ export const SUPPORTED_FILE_EXTENSIONS = [
'pl', 'pm', 'r', 'dart', 'dockerfile', 'env', 'php', 'hs', 'pl', 'pm', 'r', 'dart', 'dockerfile', 'env', 'php', 'hs',
'hsc', 'lua', 'nginxconf', 'conf', 'm', 'mm', 'plsql', 'perl', 'hsc', 'lua', 'nginxconf', 'conf', 'm', 'mm', 'plsql', 'perl',
'rb', 'rs', 'db2', 'scala', 'bash', 'swift', 'vue', 'svelte', 'rb', 'rs', 'db2', 'scala', 'bash', 'swift', 'vue', 'svelte',
'doc','docx', 'pdf', 'csv', 'txt' 'doc','docx', 'pdf', 'csv', 'txt', 'xls', 'xlsx'
]; ];
// Source: https://kit.svelte.dev/docs/modules#$env-static-public // Source: https://kit.svelte.dev/docs/modules#$env-static-public

View file

@ -12,6 +12,7 @@ export const chatId = writable('');
export const chats = writable([]); export const chats = writable([]);
export const tags = writable([]); export const tags = writable([]);
export const models = writable([]); export const models = writable([]);
export const voices = writable([]);
export const modelfiles = writable([]); export const modelfiles = writable([]);
export const prompts = writable([]); export const prompts = writable([]);

View file

@ -212,8 +212,12 @@ const convertOpenAIMessages = (convo) => {
const message = mapping[message_id]; const message = mapping[message_id];
currentId = message_id; currentId = message_id;
try { try {
if (messages.length == 0 && (message['message'] == null || if (
(message['message']['content']['parts']?.[0] == '' && message['message']['content']['text'] == null))) { messages.length == 0 &&
(message['message'] == null ||
(message['message']['content']['parts']?.[0] == '' &&
message['message']['content']['text'] == null))
) {
// Skip chat messages with no content // Skip chat messages with no content
continue; continue;
} else { } else {
@ -222,7 +226,10 @@ const convertOpenAIMessages = (convo) => {
parentId: lastId, parentId: lastId,
childrenIds: message['children'] || [], childrenIds: message['children'] || [],
role: message['message']?.['author']?.['role'] !== 'user' ? 'assistant' : 'user', role: message['message']?.['author']?.['role'] !== 'user' ? 'assistant' : 'user',
content: message['message']?.['content']?.['parts']?.[0] || message['message']?.['content']?.['text'] || '', content:
message['message']?.['content']?.['parts']?.[0] ||
message['message']?.['content']?.['text'] ||
'',
model: 'gpt-3.5-turbo', model: 'gpt-3.5-turbo',
done: true, done: true,
context: null context: null
@ -231,7 +238,7 @@ const convertOpenAIMessages = (convo) => {
lastId = currentId; lastId = currentId;
} }
} catch (error) { } catch (error) {
console.log("Error with", message, "\nError:", error); console.log('Error with', message, '\nError:', error);
} }
} }
@ -256,31 +263,31 @@ const validateChat = (chat) => {
// Because ChatGPT sometimes has features we can't use like DALL-E or migh have corrupted messages, need to validate // Because ChatGPT sometimes has features we can't use like DALL-E or migh have corrupted messages, need to validate
const messages = chat.messages; const messages = chat.messages;
// Check if messages array is empty // Check if messages array is empty
if (messages.length === 0) { if (messages.length === 0) {
return false; return false;
} }
// Last message's children should be an empty array // Last message's children should be an empty array
const lastMessage = messages[messages.length - 1]; const lastMessage = messages[messages.length - 1];
if (lastMessage.childrenIds.length !== 0) { if (lastMessage.childrenIds.length !== 0) {
return false; return false;
} }
// First message's parent should be null // First message's parent should be null
const firstMessage = messages[0]; const firstMessage = messages[0];
if (firstMessage.parentId !== null) { if (firstMessage.parentId !== null) {
return false; return false;
} }
// Every message's content should be a string // Every message's content should be a string
for (let message of messages) { for (let message of messages) {
if (typeof message.content !== 'string') { if (typeof message.content !== 'string') {
return false; return false;
} }
} }
return true; return true;
}; };
export const convertOpenAIChats = (_chats) => { export const convertOpenAIChats = (_chats) => {
@ -298,8 +305,22 @@ export const convertOpenAIChats = (_chats) => {
chat: chat, chat: chat,
timestamp: convo['timestamp'] timestamp: convo['timestamp']
}); });
} else { failed ++} } else {
failed++;
}
} }
console.log(failed, "Conversations could not be imported"); console.log(failed, 'Conversations could not be imported');
return chats; return chats;
}; };
export const isValidHttpUrl = (string) => {
let url;
try {
url = new URL(string);
} catch (_) {
return false;
}
return url.protocol === 'http:' || url.protocol === 'https:';
};

View file

@ -519,7 +519,7 @@
.filter((message) => message) .filter((message) => message)
.map((message, idx, arr) => ({ .map((message, idx, arr) => ({
role: message.role, role: message.role,
...(message.files ...(message.files?.filter((file) => file.type === 'image').length > 0 ?? false
? { ? {
content: [ content: [
{ {

View file

@ -530,7 +530,7 @@
.filter((message) => message) .filter((message) => message)
.map((message, idx, arr) => ({ .map((message, idx, arr) => ({
role: message.role, role: message.role,
...(message.files ...(message.files?.filter((file) => file.type === 'image').length > 0 ?? false
? { ? {
content: [ content: [
{ {