From b2dd2f191dbecd6b77db2348f6d13f36e48b771e Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 5 Mar 2024 01:21:50 -0800 Subject: [PATCH] refac: comment --- backend/apps/ollama/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/apps/ollama/main.py b/backend/apps/ollama/main.py index 5bd48462..fbaf622b 100644 --- a/backend/apps/ollama/main.py +++ b/backend/apps/ollama/main.py @@ -37,6 +37,11 @@ app.state.MODELS = {} REQUEST_POOL = [] +# TODO: Implement a more intelligent load balancing mechanism for distributing requests among multiple backend instances. +# Current implementation uses a simple round-robin approach (random.choice). Consider incorporating algorithms like weighted round-robin, +# least connections, or least response time for better resource utilization and performance optimization. + + @app.middleware("http") async def check_url(request: Request, call_next): if len(app.state.MODELS) == 0: @@ -761,7 +766,7 @@ async def generate_completion( @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE"]) -async def proxy(path: str, request: Request, user=Depends(get_current_user)): +async def deprecated_proxy(path: str, request: Request, user=Depends(get_current_user)): url = app.state.OLLAMA_BASE_URLS[0] target_url = f"{url}/{path}"