From e3503d66173bd6b9f89e3054a7e7ac4f2bde91df Mon Sep 17 00:00:00 2001 From: lucasew Date: Fri, 19 Jan 2024 17:12:14 -0300 Subject: [PATCH 1/5] backend: make dotenv optional Signed-off-by: lucasew --- backend/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/config.py b/backend/config.py index 2a96d018..f31ee16a 100644 --- a/backend/config.py +++ b/backend/config.py @@ -1,4 +1,3 @@ -from dotenv import load_dotenv, find_dotenv import os @@ -14,7 +13,11 @@ from constants import ERROR_MESSAGES from pathlib import Path -load_dotenv(find_dotenv("../.env")) +try: + from dotenv import load_dotenv, find_dotenv + load_dotenv(find_dotenv("../.env")) +except ImportError: + print("dotenv not installed, skipping...") #################################### From 5b26d2a686d47822d5959e90a1d0b64ebec88d67 Mon Sep 17 00:00:00 2001 From: lucasew Date: Fri, 19 Jan 2024 17:13:09 -0300 Subject: [PATCH 2/5] backend: make the data directory and the artifacts from the frontend customizable using environment variables Signed-off-by: lucasew --- backend/apps/web/internal/db.py | 4 +++- backend/apps/web/routers/utils.py | 12 +++++------- backend/config.py | 10 ++++++---- backend/main.py | 4 ++-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/backend/apps/web/internal/db.py b/backend/apps/web/internal/db.py index 3d639f3c..802d2a40 100644 --- a/backend/apps/web/internal/db.py +++ b/backend/apps/web/internal/db.py @@ -1,4 +1,6 @@ from peewee import * +from config import DATA_DIR -DB = SqliteDatabase("./data/ollama.db") + +DB = SqliteDatabase(str(DATA_DIR / "ollama.db")) DB.connect() diff --git a/backend/apps/web/routers/utils.py b/backend/apps/web/routers/utils.py index b2ca409a..cb316bbc 100644 --- a/backend/apps/web/routers/utils.py +++ b/backend/apps/web/routers/utils.py @@ -11,7 +11,7 @@ import json from utils.misc import calculate_sha256 -from config import OLLAMA_API_BASE_URL +from config import OLLAMA_API_BASE_URL, DATA_DIR, UPLOAD_DIR from constants import ERROR_MESSAGES @@ -96,8 +96,7 @@ async def download( file_name = parse_huggingface_url(url) if file_name: - os.makedirs("./uploads", exist_ok=True) - file_path = os.path.join("./uploads", f"{file_name}") + file_path = str(UPLOAD_DIR / file_name) return StreamingResponse( download_file_stream(url, file_path, file_name), @@ -109,16 +108,15 @@ async def download( @router.post("/upload") def upload(file: UploadFile = File(...)): - os.makedirs("./data/uploads", exist_ok=True) - file_path = os.path.join("./data/uploads", file.filename) + file_path = UPLOAD_DIR / file.filename # Save file in chunks - with open(file_path, "wb+") as f: + with file_path.open("wb+") as f: for chunk in file.file: f.write(chunk) def file_process_stream(): - total_size = os.path.getsize(file_path) + total_size = os.path.getsize(str(file_path)) chunk_size = 1024 * 1024 try: with open(file_path, "rb") as f: diff --git a/backend/config.py b/backend/config.py index f31ee16a..188dbbd7 100644 --- a/backend/config.py +++ b/backend/config.py @@ -24,10 +24,12 @@ except ImportError: # File Upload #################################### +DATA_DIR = Path(os.getenv("DATA_DIR", './data')).resolve() -UPLOAD_DIR = "./data/uploads" -Path(UPLOAD_DIR).mkdir(parents=True, exist_ok=True) +UPLOAD_DIR = DATA_DIR / "uploads" +UPLOAD_DIR.mkdir(parents=True, exist_ok=True) +WEB_DIR = Path(os.getenv("WEB_DIR", "../build")) #################################### # ENV (dev,test,prod) @@ -82,10 +84,10 @@ if WEBUI_AUTH and WEBUI_JWT_SECRET_KEY == "": # RAG #################################### -CHROMA_DATA_PATH = "./data/vector_db" +CHROMA_DATA_PATH = DATA_DIR / "vector_db" EMBED_MODEL = "all-MiniLM-L6-v2" CHROMA_CLIENT = chromadb.PersistentClient( - path=CHROMA_DATA_PATH, settings=Settings(allow_reset=True) + path=str(CHROMA_DATA_PATH), settings=Settings(allow_reset=True) ) CHUNK_SIZE = 1500 CHUNK_OVERLAP = 100 diff --git a/backend/main.py b/backend/main.py index e4d4bdb5..4b734da8 100644 --- a/backend/main.py +++ b/backend/main.py @@ -14,7 +14,7 @@ from apps.openai.main import app as openai_app from apps.web.main import app as webui_app from apps.rag.main import app as rag_app -from config import ENV +from config import ENV, WEB_DIR class SPAStaticFiles(StaticFiles): @@ -58,4 +58,4 @@ app.mount("/openai/api", openai_app) app.mount("/rag/api/v1", rag_app) -app.mount("/", SPAStaticFiles(directory="../build", html=True), name="spa-static-files") +app.mount("/", SPAStaticFiles(directory=str(WEB_DIR), html=True), name="spa-static-files") From d2c5f3d59164137e047f2d93f207b6f8f7a44cd3 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Mon, 22 Jan 2024 01:47:07 -0800 Subject: [PATCH 3/5] refac: convert str var to f-string --- backend/apps/web/internal/db.py | 2 +- backend/apps/web/routers/utils.py | 4 ++-- backend/config.py | 9 +++++---- backend/main.py | 8 ++++++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/backend/apps/web/internal/db.py b/backend/apps/web/internal/db.py index 802d2a40..1f8c3bf7 100644 --- a/backend/apps/web/internal/db.py +++ b/backend/apps/web/internal/db.py @@ -2,5 +2,5 @@ from peewee import * from config import DATA_DIR -DB = SqliteDatabase(str(DATA_DIR / "ollama.db")) +DB = SqliteDatabase(f"{DATA_DIR}/ollama.db") DB.connect() diff --git a/backend/apps/web/routers/utils.py b/backend/apps/web/routers/utils.py index cb316bbc..0bb3ea3c 100644 --- a/backend/apps/web/routers/utils.py +++ b/backend/apps/web/routers/utils.py @@ -96,7 +96,7 @@ async def download( file_name = parse_huggingface_url(url) if file_name: - file_path = str(UPLOAD_DIR / file_name) + file_path = f"{UPLOAD_DIR}/{file_name}" return StreamingResponse( download_file_stream(url, file_path, file_name), @@ -108,7 +108,7 @@ async def download( @router.post("/upload") def upload(file: UploadFile = File(...)): - file_path = UPLOAD_DIR / file.filename + file_path = f"{UPLOAD_DIR}/{file.filename}" # Save file in chunks with file_path.open("wb+") as f: diff --git a/backend/config.py b/backend/config.py index 188dbbd7..59ef1947 100644 --- a/backend/config.py +++ b/backend/config.py @@ -15,6 +15,7 @@ from pathlib import Path try: from dotenv import load_dotenv, find_dotenv + load_dotenv(find_dotenv("../.env")) except ImportError: print("dotenv not installed, skipping...") @@ -24,12 +25,12 @@ except ImportError: # File Upload #################################### -DATA_DIR = Path(os.getenv("DATA_DIR", './data')).resolve() +DATA_DIR = Path(os.getenv("DATA_DIR", "./data")).resolve() -UPLOAD_DIR = DATA_DIR / "uploads" +UPLOAD_DIR = f"{DATA_DIR}/uploads" UPLOAD_DIR.mkdir(parents=True, exist_ok=True) -WEB_DIR = Path(os.getenv("WEB_DIR", "../build")) +FRONTEND_BUILD_DIR = Path(os.getenv("FRONTEND_BUILD_DIR", "../build")) #################################### # ENV (dev,test,prod) @@ -84,7 +85,7 @@ if WEBUI_AUTH and WEBUI_JWT_SECRET_KEY == "": # RAG #################################### -CHROMA_DATA_PATH = DATA_DIR / "vector_db" +CHROMA_DATA_PATH = f"{DATA_DIR}/vector_db" EMBED_MODEL = "all-MiniLM-L6-v2" CHROMA_CLIENT = chromadb.PersistentClient( path=str(CHROMA_DATA_PATH), settings=Settings(allow_reset=True) diff --git a/backend/main.py b/backend/main.py index 4b734da8..a0ad73fd 100644 --- a/backend/main.py +++ b/backend/main.py @@ -14,7 +14,7 @@ from apps.openai.main import app as openai_app from apps.web.main import app as webui_app from apps.rag.main import app as rag_app -from config import ENV, WEB_DIR +from config import ENV, FRONTEND_BUILD_DIR class SPAStaticFiles(StaticFiles): @@ -58,4 +58,8 @@ app.mount("/openai/api", openai_app) app.mount("/rag/api/v1", rag_app) -app.mount("/", SPAStaticFiles(directory=str(WEB_DIR), html=True), name="spa-static-files") +app.mount( + "/", + SPAStaticFiles(directory=str(FRONTEND_BUILD_DIR), html=True), + name="spa-static-files", +) From 8da06f5e743c33389c6d24748147fbd262f40733 Mon Sep 17 00:00:00 2001 From: lucasew Date: Tue, 23 Jan 2024 12:59:52 -0300 Subject: [PATCH 4/5] fixes after the refactor Signed-off-by: lucasew --- backend/apps/web/routers/utils.py | 4 ++-- backend/config.py | 9 +++++---- backend/main.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/apps/web/routers/utils.py b/backend/apps/web/routers/utils.py index 0bb3ea3c..9adf2801 100644 --- a/backend/apps/web/routers/utils.py +++ b/backend/apps/web/routers/utils.py @@ -111,12 +111,12 @@ def upload(file: UploadFile = File(...)): file_path = f"{UPLOAD_DIR}/{file.filename}" # Save file in chunks - with file_path.open("wb+") as f: + with open(file_path, "wb+") as f: for chunk in file.file: f.write(chunk) def file_process_stream(): - total_size = os.path.getsize(str(file_path)) + total_size = os.path.getsize(file_path) chunk_size = 1024 * 1024 try: with open(file_path, "rb") as f: diff --git a/backend/config.py b/backend/config.py index 59ef1947..169baa69 100644 --- a/backend/config.py +++ b/backend/config.py @@ -25,12 +25,13 @@ except ImportError: # File Upload #################################### -DATA_DIR = Path(os.getenv("DATA_DIR", "./data")).resolve() +DATA_DIR = str(Path(os.getenv("DATA_DIR", "./data")).resolve()) UPLOAD_DIR = f"{DATA_DIR}/uploads" -UPLOAD_DIR.mkdir(parents=True, exist_ok=True) -FRONTEND_BUILD_DIR = Path(os.getenv("FRONTEND_BUILD_DIR", "../build")) +Path(UPLOAD_DIR).mkdir(parents=True, exist_ok=True) + +FRONTEND_BUILD_DIR = str(Path(os.getenv("FRONTEND_BUILD_DIR", "../build"))) #################################### # ENV (dev,test,prod) @@ -88,7 +89,7 @@ if WEBUI_AUTH and WEBUI_JWT_SECRET_KEY == "": CHROMA_DATA_PATH = f"{DATA_DIR}/vector_db" EMBED_MODEL = "all-MiniLM-L6-v2" CHROMA_CLIENT = chromadb.PersistentClient( - path=str(CHROMA_DATA_PATH), settings=Settings(allow_reset=True) + path=CHROMA_DATA_PATH, settings=Settings(allow_reset=True) ) CHUNK_SIZE = 1500 CHUNK_OVERLAP = 100 diff --git a/backend/main.py b/backend/main.py index a0ad73fd..f7a82b66 100644 --- a/backend/main.py +++ b/backend/main.py @@ -60,6 +60,6 @@ app.mount("/rag/api/v1", rag_app) app.mount( "/", - SPAStaticFiles(directory=str(FRONTEND_BUILD_DIR), html=True), + SPAStaticFiles(directory=FRONTEND_BUILD_DIR, html=True), name="spa-static-files", ) From 326af5a648c21a6833e8b3d80aaef7ff3f5bfd8b Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Thu, 25 Jan 2024 00:40:19 -0800 Subject: [PATCH 5/5] refac --- backend/config.py | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/backend/config.py b/backend/config.py index 169baa69..836a99da 100644 --- a/backend/config.py +++ b/backend/config.py @@ -1,16 +1,9 @@ import os - - import chromadb from chromadb import Settings - - from secrets import token_bytes from base64 import b64encode - from constants import ERROR_MESSAGES - - from pathlib import Path try: @@ -21,24 +14,27 @@ except ImportError: print("dotenv not installed, skipping...") -#################################### -# File Upload -#################################### - -DATA_DIR = str(Path(os.getenv("DATA_DIR", "./data")).resolve()) - -UPLOAD_DIR = f"{DATA_DIR}/uploads" - -Path(UPLOAD_DIR).mkdir(parents=True, exist_ok=True) - -FRONTEND_BUILD_DIR = str(Path(os.getenv("FRONTEND_BUILD_DIR", "../build"))) - #################################### # ENV (dev,test,prod) #################################### ENV = os.environ.get("ENV", "dev") + +#################################### +# DATA/FRONTEND BUILD DIR +#################################### + +DATA_DIR = str(Path(os.getenv("DATA_DIR", "./data")).resolve()) +FRONTEND_BUILD_DIR = str(Path(os.getenv("FRONTEND_BUILD_DIR", "../build"))) + +#################################### +# File Upload DIR +#################################### + +UPLOAD_DIR = f"{DATA_DIR}/uploads" +Path(UPLOAD_DIR).mkdir(parents=True, exist_ok=True) + #################################### # OLLAMA_API_BASE_URL ####################################