From fffded8b21f45292904159d6f58bdefa06108eca Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" <timothyjrbeck@gmail.com>
Date: Fri, 10 Nov 2023 15:16:25 -0800
Subject: [PATCH 1/3] feat: speech recognition ui added

---
 src/routes/+page.svelte | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte
index bb35c41b..e44a61fb 100644
--- a/src/routes/+page.svelte
+++ b/src/routes/+page.svelte
@@ -25,6 +25,7 @@
 		system: null,
 		temperature: null
 	};
+	let speechRecognition = true;
 
 	let models = [];
 	let chats = [];
@@ -1199,7 +1200,9 @@
 						>
 							<textarea
 								id="chat-textarea"
-								class="rounded-xl dark:bg-gray-700 dark:text-gray-100 outline-none border dark:border-gray-700 w-full py-3 px-5 pr-12 resize-none"
+								class="rounded-xl dark:bg-gray-700 dark:text-gray-100 outline-none border dark:border-gray-700 w-full py-3 {speechRecognition
+									? 'px-12'
+									: 'pl-5 pr-12'} resize-none"
 								placeholder="Send a message"
 								bind:value={prompt}
 								on:keypress={(e) => {
@@ -1216,8 +1219,32 @@
 									e.target.style.height = Math.min(e.target.scrollHeight, 200) + 2 + 'px';
 								}}
 							/>
+
+							{#if speechRecognition}
+								<div class=" absolute left-0 bottom-0">
+									<div class="pl-2.5 pb-[9px]">
+										<button
+											class="  text-gray-700 dark:text-gray-100 transition rounded-lg p-1.5"
+											on:click={() => {}}
+										>
+											<svg
+												xmlns="http://www.w3.org/2000/svg"
+												viewBox="0 0 20 20"
+												fill="currentColor"
+												class="w-5 h-5"
+											>
+												<path d="M7 4a3 3 0 016 0v6a3 3 0 11-6 0V4z" />
+												<path
+													d="M5.5 9.643a.75.75 0 00-1.5 0V10c0 3.06 2.29 5.585 5.25 5.954V17.5h-1.5a.75.75 0 000 1.5h4.5a.75.75 0 000-1.5h-1.5v-1.546A6.001 6.001 0 0016 10v-.357a.75.75 0 00-1.5 0V10a4.5 4.5 0 01-9 0v-.357z"
+												/>
+											</svg>
+										</button>
+									</div>
+								</div>
+							{/if}
+
 							<div class=" absolute right-0 bottom-0">
-								<div class="pr-3 pb-[9px]">
+								<div class="pr-2.5 pb-[9px]">
 									{#if messages.length == 0 || messages.at(-1).done == true}
 										<button
 											class="{prompt !== ''

From a7420ef4e7fb656057fe497cdbed3fce57efa721 Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" <timothyjrbeck@gmail.com>
Date: Sat, 11 Nov 2023 13:03:54 -0800
Subject: [PATCH 2/3] feat: speech recognition support

---
 src/routes/+page.svelte | 150 +++++++++++++++++++++++++++++++++++++---
 1 file changed, 140 insertions(+), 10 deletions(-)

diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte
index e44a61fb..30ca753f 100644
--- a/src/routes/+page.svelte
+++ b/src/routes/+page.svelte
@@ -25,7 +25,12 @@
 		system: null,
 		temperature: null
 	};
-	let speechRecognition = true;
+
+	let fileUploadEnabled = false;
+
+	let speechRecognition;
+	let speechRecognitionEnabled = true;
+	let speechRecognitionListening = false;
 
 	let models = [];
 	let chats = [];
@@ -187,6 +192,63 @@
 		}
 	};
 
+	const speechRecognitionHandler = () => {
+		// Check if SpeechRecognition is supported
+
+		if (speechRecognitionListening) {
+			speechRecognition.stop();
+		} else {
+			if ('SpeechRecognition' in window || 'webkitSpeechRecognition' in window) {
+				// Create a SpeechRecognition object
+				speechRecognition = new (window.SpeechRecognition || window.webkitSpeechRecognition)();
+
+				// Set continuous to true for continuous recognition
+				speechRecognition.continuous = true;
+
+				// Set the timeout for turning off the recognition after inactivity (in milliseconds)
+				const inactivityTimeout = 3000; // 3 seconds
+
+				let timeoutId;
+				// Start recognition
+				speechRecognition.start();
+				speechRecognitionListening = true;
+
+				// Event triggered when speech is recognized
+				speechRecognition.onresult = function (event) {
+					// Clear the inactivity timeout
+					clearTimeout(timeoutId);
+
+					// Handle recognized speech
+					console.log(event);
+					const transcript = event.results[Object.keys(event.results).length - 1][0].transcript;
+					prompt = `${prompt}${transcript}`;
+
+					// Restart the inactivity timeout
+					timeoutId = setTimeout(() => {
+						console.log('Speech recognition turned off due to inactivity.');
+						speechRecognition.stop();
+					}, inactivityTimeout);
+				};
+
+				// Event triggered when recognition is ended
+				speechRecognition.onend = function () {
+					// Restart recognition after it ends
+					console.log('recognition ended');
+					speechRecognitionListening = false;
+				};
+
+				// Event triggered when an error occurs
+				speechRecognition.onerror = function (event) {
+					console.log(event);
+					toast.error(`Speech recognition error: ${event.error}`);
+					speechRecognitionListening = false;
+				};
+			} else {
+				toast.error('SpeechRecognition API is not supported in this browser.');
+			}
+		}
+	};
+
 	//////////////////////////
 	// Web functions
 	//////////////////////////
@@ -1200,10 +1262,9 @@
 						>
 							<textarea
 								id="chat-textarea"
-								class="rounded-xl dark:bg-gray-700 dark:text-gray-100 outline-none border dark:border-gray-700 w-full py-3 {speechRecognition
-									? 'px-12'
-									: 'pl-5 pr-12'} resize-none"
-								placeholder="Send a message"
+								class="rounded-xl dark:bg-gray-700 dark:text-gray-100 outline-none border dark:border-gray-700 w-full py-3
+									{fileUploadEnabled ? 'pl-12' : 'pl-5'} {speechRecognitionEnabled ? 'pr-20' : 'pr-12'} resize-none"
+								placeholder={speechRecognitionListening ? 'Listening...' : 'Send a message'}
 								bind:value={prompt}
 								on:keypress={(e) => {
 									if (e.keyCode == 13 && !e.shiftKey) {
@@ -1220,12 +1281,15 @@
 								}}
 							/>
 
-							{#if speechRecognition}
+							{#if fileUploadEnabled}
 								<div class=" absolute left-0 bottom-0">
 									<div class="pl-2.5 pb-[9px]">
 										<button
-											class="  text-gray-700 dark:text-gray-100 transition rounded-lg p-1.5"
-											on:click={() => {}}
+											class="  text-gray-600 dark:text-gray-200 transition rounded-lg p-1.5"
+											type="button"
+											on:click={() => {
+												console.log('file');
+											}}
 										>
 											<svg
 												xmlns="http://www.w3.org/2000/svg"
@@ -1233,9 +1297,10 @@
 												fill="currentColor"
 												class="w-5 h-5"
 											>
-												<path d="M7 4a3 3 0 016 0v6a3 3 0 11-6 0V4z" />
 												<path
-													d="M5.5 9.643a.75.75 0 00-1.5 0V10c0 3.06 2.29 5.585 5.25 5.954V17.5h-1.5a.75.75 0 000 1.5h4.5a.75.75 0 000-1.5h-1.5v-1.546A6.001 6.001 0 0016 10v-.357a.75.75 0 00-1.5 0V10a4.5 4.5 0 01-9 0v-.357z"
+													fill-rule="evenodd"
+													d="M15.621 4.379a3 3 0 00-4.242 0l-7 7a3 3 0 004.241 4.243h.001l.497-.5a.75.75 0 011.064 1.057l-.498.501-.002.002a4.5 4.5 0 01-6.364-6.364l7-7a4.5 4.5 0 016.368 6.36l-3.455 3.553A2.625 2.625 0 119.52 9.52l3.45-3.451a.75.75 0 111.061 1.06l-3.45 3.451a1.125 1.125 0 001.587 1.595l3.454-3.553a3 3 0 000-4.242z"
+													clip-rule="evenodd"
 												/>
 											</svg>
 										</button>
@@ -1246,6 +1311,71 @@
 							<div class=" absolute right-0 bottom-0">
 								<div class="pr-2.5 pb-[9px]">
 									{#if messages.length == 0 || messages.at(-1).done == true}
+										{#if speechRecognitionEnabled}
+											<button
+												class=" text-gray-600 dark:text-gray-300 transition rounded-lg p-1.5 mr-0.5"
+												type="button"
+												on:click={() => {
+													speechRecognitionHandler();
+												}}
+											>
+												{#if speechRecognitionListening}
+													<svg
+														class=" w-5 h-5 translate-y-[1px]"
+														fill="currentColor"
+														viewBox="0 0 24 24"
+														xmlns="http://www.w3.org/2000/svg"
+														><style>
+															.spinner_qM83 {
+																animation: spinner_8HQG 1.05s infinite;
+															}
+															.spinner_oXPr {
+																animation-delay: 0.1s;
+															}
+															.spinner_ZTLf {
+																animation-delay: 0.2s;
+															}
+															@keyframes spinner_8HQG {
+																0%,
+																57.14% {
+																	animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
+																	transform: translate(0);
+																}
+																28.57% {
+																	animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
+																	transform: translateY(-6px);
+																}
+																100% {
+																	transform: translate(0);
+																}
+															}
+														</style><circle class="spinner_qM83" cx="4" cy="12" r="2.5" /><circle
+															class="spinner_qM83 spinner_oXPr"
+															cx="12"
+															cy="12"
+															r="2.5"
+														/><circle
+															class="spinner_qM83 spinner_ZTLf"
+															cx="20"
+															cy="12"
+															r="2.5"
+														/></svg
+													>
+												{:else}
+													<svg
+														xmlns="http://www.w3.org/2000/svg"
+														viewBox="0 0 20 20"
+														fill="currentColor"
+														class="w-5 h-5 translate-y-[1px]"
+													>
+														<path d="M7 4a3 3 0 016 0v6a3 3 0 11-6 0V4z" />
+														<path
+															d="M5.5 9.643a.75.75 0 00-1.5 0V10c0 3.06 2.29 5.585 5.25 5.954V17.5h-1.5a.75.75 0 000 1.5h4.5a.75.75 0 000-1.5h-1.5v-1.546A6.001 6.001 0 0016 10v-.357a.75.75 0 00-1.5 0V10a4.5 4.5 0 01-9 0v-.357z"
+														/>
+													</svg>
+												{/if}
+											</button>
+										{/if}
 										<button
 											class="{prompt !== ''
 												? 'bg-black text-white hover:bg-gray-900 dark:bg-white dark:text-black dark:hover:bg-gray-100 '

From 526fcb02c38ae6b77f65373b6b5d6a52b03426a5 Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" <timothyjrbeck@gmail.com>
Date: Sat, 11 Nov 2023 13:20:42 -0800
Subject: [PATCH 3/3] feat: speech recognition auto send toggle

---
 src/lib/components/chat/SettingsModal.svelte | 34 ++++++++++++++++++--
 src/routes/+page.svelte                      |  3 ++
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/src/lib/components/chat/SettingsModal.svelte b/src/lib/components/chat/SettingsModal.svelte
index 10363df2..e0404267 100644
--- a/src/lib/components/chat/SettingsModal.svelte
+++ b/src/lib/components/chat/SettingsModal.svelte
@@ -31,6 +31,7 @@
 	let pullProgress = null;
 
 	// Addons
+	let speechAutoSend = false;
 	let gravatarEmail = '';
 	let OPENAI_API_KEY = '';
 
@@ -89,7 +90,7 @@
 		document.documentElement.classList.add(theme);
 	};
 
-	const togglerequestFormat = async () => {
+	const toggleRequestFormat = async () => {
 		if (requestFormat === '') {
 			requestFormat = 'json';
 		} else {
@@ -99,6 +100,11 @@
 		saveSettings({ requestFormat: requestFormat !== '' ? requestFormat : undefined });
 	};
 
+	const toggleSpeechAutoSend = async () => {
+		speechAutoSend = !speechAutoSend;
+		saveSettings({ speechAutoSend: speechAutoSend });
+	};
+
 	const pullModelHandler = async () => {
 		const res = await fetch(`${API_BASE_URL}/pull`, {
 			method: 'POST',
@@ -218,8 +224,9 @@
 		top_k = settings.top_k ?? 40;
 		top_p = settings.top_p ?? 0.9;
 
-		OPENAI_API_KEY = settings.OPENAI_API_KEY ?? '';
+		speechAutoSend = settings.speechAutoSend ?? false;
 		gravatarEmail = settings.gravatarEmail ?? '';
+		OPENAI_API_KEY = settings.OPENAI_API_KEY ?? '';
 	}
 </script>
 
@@ -501,7 +508,7 @@
 									<button
 										class="p-1 px-3 text-xs flex rounded transition"
 										on:click={() => {
-											togglerequestFormat();
+											toggleRequestFormat();
 										}}
 									>
 										{#if requestFormat === ''}
@@ -740,6 +747,27 @@
 						}}
 					>
 						<div class=" space-y-3">
+							<div>
+								<div class=" py-1 flex w-full justify-between">
+									<div class=" self-center text-sm font-medium">Speech Auto-Send</div>
+
+									<button
+										class="p-1 px-3 text-xs flex rounded transition"
+										on:click={() => {
+											toggleSpeechAutoSend();
+										}}
+										type="button"
+									>
+										{#if speechAutoSend === true}
+											<span class="ml-2 self-center">On</span>
+										{:else}
+											<span class="ml-2 self-center">Off</span>
+										{/if}
+									</button>
+								</div>
+							</div>
+
+							<hr class=" dark:border-gray-700" />
 							<div>
 								<div class=" mb-2.5 text-sm font-medium">
 									Gravatar Email <span class=" text-gray-400 text-sm">(optional)</span>
diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte
index 30ca753f..98362453 100644
--- a/src/routes/+page.svelte
+++ b/src/routes/+page.svelte
@@ -235,6 +235,9 @@
 					// Restart recognition after it ends
 					console.log('recognition ended');
 					speechRecognitionListening = false;
+					if (prompt !== '' && settings?.speechAutoSend === true) {
+						submitPrompt(prompt);
+					}
 				};
 
 				// Event triggered when an error occurs