feat: youtube rag

src/lib/apis/rag/index.ts

@@ -221,6 +221,37 @@ export const uploadWebToVectorDB = async (token: string, collection_name: string
 	return res;
 };
 
+export const uploadYoutubeTranscriptionToVectorDB = async (token: string, url: string) => {
+	let error = null;
+
+	const res = await fetch(`${RAG_API_BASE_URL}/youtube`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			url: url
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
 export const queryDoc = async (
 	token: string,
 	collection_name: string,

src/lib/components/chat/MessageInput.svelte

@@ -6,7 +6,11 @@
 
 	import Prompts from './MessageInput/PromptCommands.svelte';
 	import Suggestions from './MessageInput/Suggestions.svelte';
-	import { uploadDocToVectorDB, uploadWebToVectorDB } from '$lib/apis/rag';
+	import {
+		uploadDocToVectorDB,
+		uploadWebToVectorDB,
+		uploadYoutubeTranscriptionToVectorDB
+	} from '$lib/apis/rag';
 	import AddFilesPlaceholder from '../AddFilesPlaceholder.svelte';
 	import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS } from '$lib/constants';
 	import Documents from './MessageInput/Documents.svelte';
@@ -290,6 +294,34 @@
 		}
 	};
 
+	const uploadYoutubeTranscription = async (url) => {
+		console.log(url);
+
+		const doc = {
+			type: 'doc',
+			name: url,
+			collection_name: '',
+			upload_status: false,
+			url: url,
+			error: ''
+		};
+
+		try {
+			files = [...files, doc];
+			const res = await uploadYoutubeTranscriptionToVectorDB(localStorage.token, url);
+
+			if (res) {
+				doc.upload_status = true;
+				doc.collection_name = res.collection_name;
+				files = files;
+			}
+		} catch (e) {
+			// Remove the failed doc from the files array
+			files = files.filter((f) => f.name !== url);
+			toast.error(e);
+		}
+	};
+
 	onMount(() => {
 		console.log(document.getElementById('sidebar'));
 		window.setTimeout(() => chatTextAreaElement?.focus(), 0);
@@ -428,6 +460,10 @@
 						<Documents
 							bind:this={documentsElement}
 							bind:prompt
+							on:youtube={(e) => {
+								console.log(e);
+								uploadYoutubeTranscription(e.detail);
+							}}
 							on:url={(e) => {
 								console.log(e);
 								uploadWeb(e.detail);

src/lib/components/chat/MessageInput/Documents.svelte

@@ -87,6 +87,17 @@
 		chatInputElement?.focus();
 		await tick();
 	};
+
+	const confirmSelectYoutube = async (url) => {
+		dispatch('youtube', url);
+
+		prompt = removeFirstHashWord(prompt);
+		const chatInputElement = document.getElementById('chat-textarea');
+
+		await tick();
+		chatInputElement?.focus();
+		await tick();
+	};
 </script>
 
 {#if filteredItems.length > 0 || prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
@@ -132,7 +143,30 @@
 						</button>
 					{/each}
 
-					{#if prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
+					{#if prompt.split(' ')?.at(0)?.substring(1).startsWith('https://www.youtube.com')}
+						<button
+							class="px-3 py-1.5 rounded-xl w-full text-left bg-gray-100 selected-command-option-button"
+							type="button"
+							on:click={() => {
+								const url = prompt.split(' ')?.at(0)?.substring(1);
+								if (isValidHttpUrl(url)) {
+									confirmSelectYoutube(url);
+								} else {
+									toast.error(
+										$i18n.t(
+											'Oops! Looks like the URL is invalid. Please double-check and try again.'
+										)
+									);
+								}
+							}}
+						>
+							<div class=" font-medium text-black line-clamp-1">
+								{prompt.split(' ')?.at(0)?.substring(1)}
+							</div>
+
+							<div class=" text-xs text-gray-600 line-clamp-1">{$i18n.t('Youtube')}</div>
+						</button>
+					{:else if prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
 						<button
 							class="px-3 py-1.5 rounded-xl w-full text-left bg-gray-100 selected-command-option-button"
 							type="button"