From 0d498cfa30fd871d961b088e79608bb8f47808fe Mon Sep 17 00:00:00 2001 From: Sebastian Ovide Date: Sun, 30 Jun 2024 20:39:53 +0100 Subject: [PATCH] text to speach via capacitor plugin --- package-lock.json | 343 +-------------------------------- package.json | 1 + src/js/audio/sound.js | 76 ++++---- src/js/visual/voicecontrols.js | 60 ++++-- 4 files changed, 92 insertions(+), 388 deletions(-) diff --git a/package-lock.json b/package-lock.json index 122148c..3d3d1c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "@capacitor-community/text-to-speech": "^4.0.2", "@capacitor/camera": "latest", "@capacitor/core": "latest", "@capacitor/splash-screen": "latest", @@ -1740,6 +1741,14 @@ "node": ">=6.9.0" } }, + "node_modules/@capacitor-community/text-to-speech": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@capacitor-community/text-to-speech/-/text-to-speech-4.0.2.tgz", + "integrity": "sha512-lpkcCUGpBJEAOyR6biYsM/gGkq50ZFupLhO5HNyiACCa2QkR9uRqNHByMUAtvg/nFqlAIg/CVJ24Odn68JhKzw==", + "peerDependencies": { + "@capacitor/core": "^6.0.0" + } + }, "node_modules/@capacitor/camera": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/@capacitor/camera/-/camera-6.0.1.tgz", @@ -1818,22 +1827,6 @@ "@capacitor/core": "^6.0.0" } }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.14.54.tgz", - "integrity": "sha512-bZBrLAIX1kpWelV0XemxBZllyRmM6vgFQQG2GdNb+r3Fkp0FOh1NJSvekXDs7jq70k4euu1cryLMfU+mTXlEpw==", - "cpu": [ - "loong64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, "node_modules/@ionic/cli-framework-output": { "version": "2.2.8", "resolved": "https://registry.npmjs.org/@ionic/cli-framework-output/-/cli-framework-output-2.2.8.tgz", @@ -4815,118 +4808,6 @@ "esbuild-windows-arm64": "0.14.54" } }, - "node_modules/esbuild-android-64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.14.54.tgz", - "integrity": "sha512-Tz2++Aqqz0rJ7kYBfz+iqyE3QMycD4vk7LBRyWaAVFgFtQ/O8EJOnVmTOiDWYZ/uYzB4kvP+bqejYdVKzE5lAQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-android-arm64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.14.54.tgz", - "integrity": "sha512-F9E+/QDi9sSkLaClO8SOV6etqPd+5DgJje1F9lOWoNncDdOBL2YF59IhsWATSt0TLZbYCf3pNlTHvVV5VfHdvg==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-darwin-64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.14.54.tgz", - "integrity": "sha512-jtdKWV3nBviOd5v4hOpkVmpxsBy90CGzebpbO9beiqUYVMBtSc0AL9zGftFuBon7PNDcdvNCEuQqw2x0wP9yug==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-darwin-arm64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.54.tgz", - "integrity": "sha512-OPafJHD2oUPyvJMrsCvDGkRrVCar5aVyHfWGQzY1dWnzErjrDuSETxwA2HSsyg2jORLY8yBfzc1MIpUkXlctmw==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-freebsd-64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.54.tgz", - "integrity": "sha512-OKwd4gmwHqOTp4mOGZKe/XUlbDJ4Q9TjX0hMPIDBUWWu/kwhBAudJdBoxnjNf9ocIB6GN6CPowYpR/hRCbSYAg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-freebsd-arm64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.54.tgz", - "integrity": "sha512-sFwueGr7OvIFiQT6WeG0jRLjkjdqWWSrfbVwZp8iMP+8UHEHRBvlaxL6IuKNDwAozNUmbb8nIMXa7oAOARGs1Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-linux-32": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.14.54.tgz", - "integrity": "sha512-1ZuY+JDI//WmklKlBgJnglpUL1owm2OX+8E1syCD6UAxcMM/XoWd76OHSjl/0MR0LisSAXDqgjT3uJqT67O3qw==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, "node_modules/esbuild-linux-64": { "version": "0.14.54", "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.14.54.tgz", @@ -4943,198 +4824,6 @@ "node": ">=12" } }, - "node_modules/esbuild-linux-arm": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.14.54.tgz", - "integrity": "sha512-qqz/SjemQhVMTnvcLGoLOdFpCYbz4v4fUo+TfsWG+1aOu70/80RV6bgNpR2JCrppV2moUQkww+6bWxXRL9YMGw==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-linux-arm64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.54.tgz", - "integrity": "sha512-WL71L+0Rwv+Gv/HTmxTEmpv0UgmxYa5ftZILVi2QmZBgX3q7+tDeOQNqGtdXSdsL8TQi1vIaVFHUPDe0O0kdig==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-linux-mips64le": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.54.tgz", - "integrity": "sha512-qTHGQB8D1etd0u1+sB6p0ikLKRVuCWhYQhAHRPkO+OF3I/iSlTKNNS0Lh2Oc0g0UFGguaFZZiPJdJey3AGpAlw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-linux-ppc64le": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.54.tgz", - "integrity": "sha512-j3OMlzHiqwZBDPRCDFKcx595XVfOfOnv68Ax3U4UKZ3MTYQB5Yz3X1mn5GnodEVYzhtZgxEBidLWeIs8FDSfrQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-linux-riscv64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.14.54.tgz", - "integrity": "sha512-y7Vt7Wl9dkOGZjxQZnDAqqn+XOqFD7IMWiewY5SPlNlzMX39ocPQlOaoxvT4FllA5viyV26/QzHtvTjVNOxHZg==", - "cpu": [ - "riscv64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-linux-s390x": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.54.tgz", - "integrity": "sha512-zaHpW9dziAsi7lRcyV4r8dhfG1qBidQWUXweUjnw+lliChJqQr+6XD71K41oEIC3Mx1KStovEmlzm+MkGZHnHA==", - "cpu": [ - "s390x" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-netbsd-64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.54.tgz", - "integrity": "sha512-PR01lmIMnfJTgeU9VJTDY9ZerDWVFIUzAtJuDHwwceppW7cQWjBBqP48NdeRtoP04/AtO9a7w3viI+PIDr6d+w==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-openbsd-64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.54.tgz", - "integrity": "sha512-Qyk7ikT2o7Wu76UsvvDS5q0amJvmRzDyVlL0qf5VLsLchjCa1+IAvd8kTBgUxD7VBUUVgItLkk609ZHUc1oCaw==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-sunos-64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.14.54.tgz", - "integrity": "sha512-28GZ24KmMSeKi5ueWzMcco6EBHStL3B6ubM7M51RmPwXQGLe0teBGJocmWhgwccA1GeFXqxzILIxXpHbl9Q/Kw==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-windows-32": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.14.54.tgz", - "integrity": "sha512-T+rdZW19ql9MjS7pixmZYVObd9G7kcaZo+sETqNH4RCkuuYSuv9AGHUVnPoP9hhuE1WM1ZimHz1CIBHBboLU7w==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-windows-64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.14.54.tgz", - "integrity": "sha512-AoHTRBUuYwXtZhjXZbA1pGfTo8cJo3vZIcWGLiUcTNgHpJJMC1rVA44ZereBHMJtotyN71S8Qw0npiCIkW96cQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/esbuild-windows-arm64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.54.tgz", - "integrity": "sha512-M0kuUvXhot1zOISQGXwWn6YtS+Y/1RT9WrVIOywZnJHo3jCDyewAc79aKNQWFCQm+xNHVTq9h8dZKvygoXQQRg==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, "node_modules/escalade": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", @@ -5447,20 +5136,6 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", diff --git a/package.json b/package.json index 0b9a0fa..517b945 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "serve-tiles": "node server.js" }, "dependencies": { + "@capacitor-community/text-to-speech": "^4.0.2", "@capacitor/camera": "latest", "@capacitor/core": "latest", "@capacitor/splash-screen": "latest", diff --git a/src/js/audio/sound.js b/src/js/audio/sound.js index 5e0835f..793877d 100644 --- a/src/js/audio/sound.js +++ b/src/js/audio/sound.js @@ -1,13 +1,15 @@ // Copyright (c) Daniel W. Steinbrook. // with many thanks to ChatGPT -import { createPanner } from './notabeacon.js' +import { TextToSpeech } from "@capacitor-community/text-to-speech"; +import { createPanner } from "./notabeacon.js"; -export const audioContext = new (window.AudioContext || window.webkitAudioContext)(); +export const audioContext = new (window.AudioContext || + window.webkitAudioContext)(); // Variables to store the current sound and speech sources let currentSoundSource = null; -let currentSpeechSource = null; +// let currentSpeechSource = null; // Fetch and decode each sound effect only once, and store here by URL let audioBufferCache = {}; @@ -21,7 +23,7 @@ async function loadSound(url) { const arrayBuffer = await response.arrayBuffer(); audioBufferCache[url] = await audioContext.decodeAudioData(arrayBuffer); } catch (error) { - console.error('Error loading sound:', error); + console.error("Error loading sound:", error); return; } } @@ -63,26 +65,11 @@ function playSpatialSound(buffer, x, y) { // Function to play synthesized speech with spatial audio //FIXME not actually spatial -export function playSpatialSpeech(text, voice, rate, x, y) { +export async function playSpatialSpeech(text, voice, rate, x, y) { // Cancel the current speech source if any - if (currentSpeechSource) { - speechSynthesis.cancel(); - } - - return new Promise((resolve) => { - const utterance = new SpeechSynthesisUtterance(text); - if (rate) { - utterance.rate = rate; - } - if (voice) { - utterance.voice = voice; - } - utterance.onend = () => resolve(); - speechSynthesis.speak(utterance); + TextToSpeech.stop(); - // Update the current speech source - currentSpeechSource = utterance; - }); + return TextToSpeech.speak({ text, voice, rate }); } // Function to create a player with a dynamic sequence of spatial sounds and spatial speech @@ -130,12 +117,7 @@ export function createSpatialPlayer(locationProvider) { player.isPlaying = false; // Cancel the current sound and speech sources - if (currentSoundSource) { - currentSoundSource.stop(); - } - if (currentSpeechSource) { - speechSynthesis.cancel(); - } + TextToSpeech.stop(); }, }; @@ -150,28 +132,44 @@ export function createSpatialPlayer(locationProvider) { // Calculate the Cartesian coordinates to position the audio. // (done just before the audio is spoken, since the user may have // moved since the audio was queued) - var relativePosition = {x: 0, y: 0}; + var relativePosition = { x: 0, y: 0 }; if (currentItem.location) { - relativePosition = player.locationProvider.normalizedRelativePosition(currentItem.location); + relativePosition = player.locationProvider.normalizedRelativePosition( + currentItem.location + ); } // Compute current distance to POI (may be greater than proximityThreshold, if user has moved away since it was queued) if (currentItem.includeDistance) { - const units = 'feet'; - const distance = player.locationProvider.distance(currentItem.location, { units: units }).toFixed(0); - currentItem.text += `, ${distance} ${units}` + const units = "feet"; + const distance = player.locationProvider + .distance(currentItem.location, { units: units }) + .toFixed(0); + currentItem.text += `, ${distance} ${units}`; } - if (typeof currentItem === 'object' && currentItem.soundUrl) { + if (typeof currentItem === "object" && currentItem.soundUrl) { // If it's an object with a 'soundUrl' property, assume it's a spatial sound const soundBuffer = await loadSound(currentItem.soundUrl); - await playSpatialSound(soundBuffer, relativePosition.x || 0, relativePosition.y || 0); - } else if (typeof currentItem === 'object' && currentItem.text) { + await playSpatialSound( + soundBuffer, + relativePosition.x || 0, + relativePosition.y || 0 + ); + } else if (typeof currentItem === "object" && currentItem.text) { // If it's an object with a 'text' property, assume it's spatial speech - player.events.dispatchEvent(new CustomEvent('speechPlayed', { detail: currentItem })); - await playSpatialSpeech(currentItem.text, player.voice, player.rate, relativePosition.x || 0, relativePosition.y || 0); + player.events.dispatchEvent( + new CustomEvent("speechPlayed", { detail: currentItem }) + ); + await playSpatialSpeech( + currentItem.text, + player.voice, + player.rate, + relativePosition.x || 0, + relativePosition.y || 0 + ); } else { - console.error(`unrecognized object in audio queue: ${currentItem}`) + console.error(`unrecognized object in audio queue: ${currentItem}`); } // Play the next item recursively diff --git a/src/js/visual/voicecontrols.js b/src/js/visual/voicecontrols.js index 17643bd..8b7a39a 100644 --- a/src/js/visual/voicecontrols.js +++ b/src/js/visual/voicecontrols.js @@ -1,37 +1,67 @@ // Copyright (c) Daniel W. Steinbrook. // with many thanks to ChatGPT +import { TextToSpeech } from "@capacitor-community/text-to-speech"; function createVoiceControls(audioQueue) { // Fetch available voices - const voiceSelect = document.getElementById('voice'); + const voiceSelect = document.getElementById("voice"); // const rateInput = document.getElementById('rate'); - const decreaseRate = document.getElementById('decreaseRate'); - const increaseRate = document.getElementById('increaseRate'); - const rateValue = document.getElementById('rateValue'); + const decreaseRate = document.getElementById("decreaseRate"); + const increaseRate = document.getElementById("increaseRate"); + const rateValue = document.getElementById("rateValue"); + + // Just for testing + // TextToSpeech.speak({ + // text: "This is a sample text.", + // lang: "en-US", + // rate: 1.0, + // pitch: 1.0, + // volume: 1.0, + // category: "ambient", + // }); // Populate voice selector function populateVoices() { // Populate voice list with all English voices - audioQueue.voices = window.speechSynthesis.getVoices() - .filter(voice => voice.lang.startsWith('en'));; - audioQueue.voices.forEach(function(voice, index) { - const option = document.createElement('option'); - option.value = index; - option.textContent = '🗣 ' + voice.name; - voiceSelect.appendChild(option); + audioQueue.voices = []; + + TextToSpeech.getSupportedVoices().then((voices) => { + const voicesEn = voices.voices.filter((voice) => + voice.lang.startsWith("en") + ); + const voicesNames = new Set(voicesEn.map((voice) => voice.name)); + + audioQueue.voices = Array.from(voicesNames).map((name) => + voicesEn.find((voice) => voice.name === name) + ); + + // Remove them to avoid duplicates + while (voiceSelect.childNodes[0] != null) { + voiceSelect.childNodes[0].remove(); + } + + // console.log(`I'll add ${audioQueue.voices.length} voices to the list`); + audioQueue.voices.forEach(function (voice, index) { + const option = document.createElement("option"); + option.value = index; + option.textContent = "🗣 " + voice.name; + voiceSelect.appendChild(option); + + console.log(`VOICE ${voice.name} with value ${index} APPENDED`); + }); }); } populateVoices(); // Select the system default voice by default - const systemDefaultVoice = audioQueue.voices.find(voice => voice.default); + const systemDefaultVoice = audioQueue.voices.find((voice) => voice.default); if (systemDefaultVoice) { voiceSelect.value = audioQueue.voices.indexOf(systemDefaultVoice); } // Update voices when they change - window.speechSynthesis.onvoiceschanged = function() { - voiceSelect.innerHTML = ''; // Clear existing options + window.speechSynthesis.onvoiceschanged = function () { + voiceSelect.innerHTML = ""; // Clear existing options populateVoices(); }; @@ -44,7 +74,7 @@ function createVoiceControls(audioQueue) { rateValue.textContent = audioQueue.increaseRate(); }); - voiceSelect.addEventListener('change', function() { + voiceSelect.addEventListener("change", function () { audioQueue.setVoice(voiceSelect.value); });