Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ca2415c77d | ||
|
|
322a93b5d9 | ||
|
|
1c8d60e6be | ||
|
|
a84cefbc57 | ||
|
|
a860b06b3b | ||
|
|
d9ef95f2c6 | ||
|
|
3b4a6d6484 | ||
|
|
e97146cb38 | ||
|
|
75c19b915b | ||
|
|
99e1c62c1e | ||
|
|
6b076bd6db | ||
|
|
0130d03bf9 | ||
|
|
1f1578b163 | ||
|
|
c1bba5473c | ||
|
|
3a735d244f | ||
|
|
f95798954a | ||
|
|
3e8155b4bb | ||
|
|
31ac177d69 | ||
|
|
491b6160a8 | ||
|
|
4992d0badd | ||
|
|
38f27e68d6 | ||
|
|
df186c50c6 | ||
|
|
e675978573 | ||
|
|
0766d55a54 | ||
|
|
c8567845c3 |
@ -3,7 +3,6 @@ NEXT_PUBLIC_AMICA_API_URL=https://store.heyamica.com
|
||||
NEXT_PUBLIC_AMICA_STORAGE_URL=https://vrm.heyamica.com/file/amica-vrm
|
||||
NEXT_PUBLIC_SUPABASE_URL=https://kzfcwefqffysqsjrlyld.supabase.co
|
||||
NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Imt6ZmN3ZWZxZmZ5c3FzanJseWxkIiwicm9sZSI6ImFub24iLCJpYXQiOjE3MDEyNzA5NjUsImV4cCI6MjAxNjg0Njk2NX0.z9zm2mbJu6RkdZ6zkCUltThIF43-ava_bLvk9qFQsiA
|
||||
NEXT_PUBLIC_WALLETCONNECT_PROJECT_ID=9f8cee238fde4f0011880ecef0616c88
|
||||
|
||||
X_API_KEY="X_API_KEY"
|
||||
X_API_SECRET="X_API_SECRET"
|
||||
|
||||
@ -35,6 +35,7 @@
|
||||
* [Using Coqui Local](./guides/using-coqui.md)
|
||||
* [Using Piper ](./guides/using-piper.md)
|
||||
* [Using Alltalk TTS](./guides/using-alltalk.md)
|
||||
* [Using Kokoro TTS](./guides/using-kokoro.md)
|
||||
* [Using RVC](./guides/using-rvc.md)
|
||||
|
||||
|
||||
|
||||
51
docs/guides/using-kokoro.md
Normal file
51
docs/guides/using-kokoro.md
Normal file
@ -0,0 +1,51 @@
|
||||
---
|
||||
title: Using Kokoro
|
||||
order: 16
|
||||
---
|
||||
|
||||
Navigate to [Kokoro TTS GitHub repository](https://github.com/hexgrad/kokoro).
|
||||
|
||||
## Setting Up Kokoro TTS Server
|
||||
|
||||
### Clone the Repository
|
||||
```bash
|
||||
git clone https://github.com/flukexp/kokoro-tts.git
|
||||
cd kokoro-tts
|
||||
```
|
||||
|
||||
### Create a Virtual Environment
|
||||
```bash
|
||||
python -m venv venv
|
||||
source venv/bin/activate # On Windows, use `venv\Scripts\activate`
|
||||
```
|
||||
|
||||
### Install Dependencies
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
## Running the Server
|
||||
|
||||
### Start the FastAPI Server
|
||||
```bash
|
||||
python server.py
|
||||
```
|
||||
|
||||
## Make sure Kokoro is enabled for TTS:
|
||||
|
||||
```bash
|
||||
Settings -> Text-to-Speech -> TTS Backend -> Kokoro
|
||||
```
|
||||
|
||||
## Set the voice
|
||||
|
||||
```bash
|
||||
Settings -> Text-to-Speech -> Kokoro -> Voice
|
||||
```
|
||||
|
||||
## Using Kokoro with OpenAI TTS
|
||||
You can use Kokoro by choosing OpenAI and configuring your Kokoro endpoint and voice.
|
||||
|
||||
### Notes
|
||||
- Kokoro TTS can be used as a local text-to-speech backend in your application.
|
||||
- If you want to explore more models or functionalities, refer to the official [Kokoro TTS GitHub repository](https://github.com/hexgrad/kokoro).
|
||||
@ -8,7 +8,7 @@ To test Moshi, you need to set up and run the Moshi Server on Runpod (Or your ru
|
||||
|
||||
---
|
||||
|
||||
### **Step 1: Set Up and Running Moshi Server for Amica**
|
||||
### **Step 1: Set Up Moshi Server**
|
||||
1. Login to the Terminal on your instance, whether it is your own server or a Runpod instance (If you don't have a good GPU)
|
||||
1. **Clone the Moshi Server**
|
||||
```bash
|
||||
@ -34,9 +34,14 @@ To test Moshi, you need to set up and run the Moshi Server on Runpod (Or your ru
|
||||
.\venv\Scripts\Activate
|
||||
```
|
||||
|
||||
4. **Install dependencies**
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### **Step 2: Run the Moshi Server**
|
||||
### **Step 2: Start the Moshi Server**
|
||||
|
||||
Start the Moshi server (For Runpod):
|
||||
```bash
|
||||
@ -64,6 +69,6 @@ Then go to Settings > Chatbot Backend > Moshi, and then insert the correct URL f
|
||||
- Ensure that **Python and pip** are installed before proceeding. **Edit the model URLs in the main python script if you want to use a different model from Kyutai.**
|
||||
---
|
||||
|
||||
Replace the `{port-number}` in the URL with the actual port number (e.g 3000 for Amica and 8000 for Moshi).
|
||||
Replace the `{port-number}` in the URL with the actual port number.
|
||||
|
||||
---
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
// run i18next
|
||||
export default {
|
||||
defaultNamespace: 'common',
|
||||
locales: ['en', 'zh', 'de'],
|
||||
locales: ['en', 'zh', 'de', 'ka'],
|
||||
output: 'src/i18n/locales/$LOCALE/$NAMESPACE.json',
|
||||
input: ["src/**/*.{ts,tsx}"],
|
||||
keepRemoved: true,
|
||||
|
||||
10422
package-lock.json
generated
10422
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -28,13 +28,11 @@
|
||||
"@mkkellogg/gaussian-splats-3d": "^0.4.5",
|
||||
"@next/third-parties": "^14.2.16",
|
||||
"@pixiv/three-vrm": "^3.1.2",
|
||||
"@rainbow-me/rainbowkit": "^2.2.0",
|
||||
"@ricky0123/vad-react": "^0.0.24",
|
||||
"@sentry/nextjs": "^8.34.0",
|
||||
"@supabase/supabase-js": "^2.45.4",
|
||||
"@tabler/icons-react": "^3.19.0",
|
||||
"@tailwindcss/forms": "^0.5.9",
|
||||
"@tanstack/react-query": "^5.59.16",
|
||||
"@testing-library/jest-dom": "^6.5.0",
|
||||
"@webxr-input-profiles/assets": "^1.0.16",
|
||||
"@webxr-input-profiles/motion-controllers": "^1.0.0",
|
||||
@ -66,8 +64,6 @@
|
||||
"three.quarks": "^0.15.5",
|
||||
"twitter-api-v2": "^1.19.0",
|
||||
"typescript-collections": "^1.3.3",
|
||||
"viem": "^2.21.37",
|
||||
"wagmi": "^2.12.25",
|
||||
"wavefile": "^11.0.0",
|
||||
"webm-duration-fix": "^1.0.4",
|
||||
"window.ai": "^0.2.4",
|
||||
|
||||
@ -17,7 +17,7 @@ class Scenario {
|
||||
|
||||
await $.loadVrm(
|
||||
// 'https://vrm.heyamica.com/file/amica-vrm/82754e287e0b26b5d7a1fd223ed0fd5debcabed81f36549fb2c16b201f3e5ca9',
|
||||
'/vrm/AvatarSample_A.vrm',
|
||||
'/vrm/AvatarSample_B.vrm',
|
||||
(progress) => {
|
||||
console.log(`loading model ${progress}`);
|
||||
},
|
||||
@ -49,6 +49,6 @@ class Scenario {
|
||||
}
|
||||
|
||||
update(delta) {
|
||||
// console.log('update', this.x);
|
||||
console.log('update', this.x);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,3 @@
|
||||
import { useAccount } from 'wagmi'
|
||||
import { ConnectButton } from '@rainbow-me/rainbowkit';
|
||||
import { useState } from "react";
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
export const ArbiusIntroduction = ({ open, close }: {
|
||||
@ -8,10 +5,8 @@ export const ArbiusIntroduction = ({ open, close }: {
|
||||
close: () => void;
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const { address } = useAccount();
|
||||
|
||||
|
||||
if (! open) {
|
||||
if (!open) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -29,23 +24,15 @@ export const ArbiusIntroduction = ({ open, close }: {
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="my-8 flex space-x-4">
|
||||
<ConnectButton />
|
||||
</div>
|
||||
<div className="my-8">
|
||||
{address && (
|
||||
<button
|
||||
onClick={() => {
|
||||
close();
|
||||
}}
|
||||
className="ml-3 inline-flex items-center rounded-md bg-white px-2.5 py-1.5 text-sm font-semibold text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 hover:bg-gray-50"
|
||||
>
|
||||
{t("Continue to Amica")}
|
||||
</button>
|
||||
)}
|
||||
{(!address) && (
|
||||
<p>{t("Please connect your wallet to continue.")}</p>
|
||||
)}
|
||||
<button
|
||||
onClick={() => {
|
||||
close();
|
||||
}}
|
||||
className="ml-3 inline-flex items-center rounded-md bg-white px-2.5 py-1.5 text-sm font-semibold text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 hover:bg-gray-50"
|
||||
>
|
||||
{t("Continue to Amica")}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -14,6 +14,7 @@ import { whispercpp } from "@/features/whispercpp/whispercpp";
|
||||
import { config } from "@/utils/config";
|
||||
import { WaveFile } from "wavefile";
|
||||
import { AmicaLifeContext } from "@/features/amicaLife/amicaLifeContext";
|
||||
import { AudioControlsContext } from "@/features/moshi/components/audioControlsContext";
|
||||
|
||||
|
||||
export default function MessageInput({
|
||||
@ -36,6 +37,8 @@ export default function MessageInput({
|
||||
const { chat: bot } = useContext(ChatContext);
|
||||
const { alert } = useContext(AlertContext);
|
||||
const { amicaLife } = useContext(AmicaLifeContext);
|
||||
const { audioControls: moshi } = useContext(AudioControlsContext);
|
||||
const [ moshiMuted, setMoshiMuted] = useState(moshi.isMuted());
|
||||
|
||||
const vad = useMicVAD({
|
||||
startOnLoad: false,
|
||||
@ -212,20 +215,33 @@ export default function MessageInput({
|
||||
<div className="grid grid-flow-col grid-cols-[min-content_1fr_min-content] gap-[8px]">
|
||||
<div>
|
||||
<div className='flex flex-col justify-center items-center'>
|
||||
<IconButton
|
||||
{config("chatbot_backend") === "moshi" ? (
|
||||
<IconButton
|
||||
iconName={!moshiMuted ? "24/PauseAlt" : "24/Microphone"}
|
||||
className="bg-secondary hover:bg-secondary-hover active:bg-secondary-press disabled:bg-secondary-disabled"
|
||||
isProcessing={moshiMuted && moshi.getRecorder() != null}
|
||||
disabled={!moshi.getRecorder()}
|
||||
onClick={() => {
|
||||
moshi.toggleMute();
|
||||
setMoshiMuted(!moshiMuted);
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<IconButton
|
||||
iconName={vad.listening ? "24/PauseAlt" : "24/Microphone"}
|
||||
className="bg-secondary hover:bg-secondary-hover active:bg-secondary-press disabled:bg-secondary-disabled"
|
||||
isProcessing={vad.userSpeaking}
|
||||
disabled={config('stt_backend') === 'none' || vad.loading || Boolean(vad.errored)}
|
||||
onClick={vad.toggle}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<input
|
||||
type="text"
|
||||
ref={inputRef}
|
||||
placeholder="Write message here..."
|
||||
placeholder={config("chatbot_backend") === "moshi" ? "Disabled in moshi chatbot" : "Write message here..."}
|
||||
onChange={handleInputChange}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter") {
|
||||
@ -240,7 +256,7 @@ export default function MessageInput({
|
||||
clickedSendButton();
|
||||
}
|
||||
}}
|
||||
disabled={false}
|
||||
disabled={config("chatbot_backend") === "moshi"}
|
||||
|
||||
className="disabled block w-full rounded-md border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-1 focus:ring-inset focus:ring-gray-400 sm:text-sm sm:leading-6"
|
||||
value={userMessage}
|
||||
@ -252,7 +268,7 @@ export default function MessageInput({
|
||||
iconName="24/Send"
|
||||
className="ml-2 bg-secondary hover:bg-secondary-hover active:bg-secondary-press disabled:bg-secondary-disabled"
|
||||
isProcessing={isChatProcessing || transcriber.isBusy}
|
||||
disabled={isChatProcessing || !userMessage || transcriber.isModelLoading}
|
||||
disabled={isChatProcessing || !userMessage || transcriber.isModelLoading || config("chatbot_backend") === "moshi"}
|
||||
onClick={clickedSendButton}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@ -72,6 +72,7 @@ import { AmicaLifePage } from "./settings/AmicaLifePage";
|
||||
import { useVrmStoreContext } from "@/features/vrmStore/vrmStoreContext";
|
||||
import { OpenRouterSettings } from "./settings/OpenRouterSettingsPage";
|
||||
import { ExternalAPIPage } from "./settings/ExternalAPIPage";
|
||||
import { KokoroSettingsPage } from "./settings/KokoroSettingsPage";
|
||||
|
||||
|
||||
export const Settings = ({
|
||||
@ -135,6 +136,9 @@ export const Settings = ({
|
||||
|
||||
const [localXTTSUrl, setLocalXTTSUrl] = useState(config("localXTTS_url"));
|
||||
|
||||
const [kokoroUrl, setKokoroUrl] = useState(config("kokoro_url"));
|
||||
const [kokoroVoice, setKokoroVoice] = useState(config("kokoro_voice"));
|
||||
|
||||
const [visionBackend, setVisionBackend] = useState(config("vision_backend"));
|
||||
const [visionLlamaCppUrl, setVisionLlamaCppUrl] = useState(config("vision_llamacpp_url"));
|
||||
const [visionOllamaUrl, setVisionOllamaUrl] = useState(config("vision_ollama_url"));
|
||||
@ -252,7 +256,7 @@ export const Settings = ({
|
||||
|
||||
useEffect(() => {
|
||||
// Change the chatbot to 'llamacpp' if Amica Life is enabled and previous chatbot was 'echo'
|
||||
if (amicaLifeEnabled && chatbotBackend === "echo") {
|
||||
if (amicaLifeEnabled && ["echo", "moshi"].includes(config("chatbot_backend"))) {
|
||||
setAmicaLifeEnabled(false);
|
||||
}
|
||||
}, [chatbotBackend, amicaLifeEnabled]);
|
||||
@ -284,6 +288,7 @@ export const Settings = ({
|
||||
rvcUrl,rvcEnabled,rvcModelName,rvcIndexPath,rvcF0upKey,rvcF0Method,rvcIndexRate,rvcFilterRadius,,rvcResampleSr,rvcRmsMixRate,rvcProtect,
|
||||
coquiLocalUrl,coquiLocalVoiceId,
|
||||
localXTTSUrl,
|
||||
kokoroUrl, kokoroVoice,
|
||||
visionBackend,
|
||||
visionLlamaCppUrl,
|
||||
visionOllamaUrl, visionOllamaModel,
|
||||
@ -368,7 +373,7 @@ export const Settings = ({
|
||||
|
||||
case 'tts':
|
||||
return <MenuPage
|
||||
keys={["tts_backend", "elevenlabs_settings", "speecht5_settings", "coquiLocal_settings", "openai_tts_settings", "piper_settings", "localXTTS_settings", "rvc_settings"]}
|
||||
keys={["tts_backend", "elevenlabs_settings", "speecht5_settings", "coquiLocal_settings", "openai_tts_settings", "piper_settings", "localXTTS_settings", "kokoro_settings", "rvc_settings"]}
|
||||
menuClick={handleMenuClick} />;
|
||||
|
||||
case 'stt':
|
||||
@ -583,6 +588,15 @@ export const Settings = ({
|
||||
setSettingsUpdated={setSettingsUpdated}
|
||||
/>
|
||||
|
||||
case 'kokoro_settings':
|
||||
return <KokoroSettingsPage
|
||||
kokoroUrl={kokoroUrl}
|
||||
kokoroVoice={kokoroVoice}
|
||||
setKokoroVoice={setKokoroVoice}
|
||||
setKokoroUrl={setKokoroUrl}
|
||||
setSettingsUpdated={setSettingsUpdated}
|
||||
/>
|
||||
|
||||
case 'rvc_settings':
|
||||
return <RVCSettingsPage
|
||||
rvcUrl={rvcUrl}
|
||||
|
||||
@ -99,7 +99,7 @@ export function AmicaLifePage({
|
||||
<SwitchBox
|
||||
value={amicaLifeEnabled}
|
||||
label={`${t("Amica Life")} ${t("Enabled")} ${t("(Disable to improve performance)")}`}
|
||||
disabled={config("chatbot_backend") === "echo"}
|
||||
disabled={["echo", "moshi"].includes(config("chatbot_backend")) }
|
||||
onChange={(value: boolean) => {
|
||||
setAmicaLifeEnabled(value);
|
||||
updateConfig("amica_life_enabled", value.toString());
|
||||
|
||||
@ -61,6 +61,7 @@ export function CoquiLocalSettingsPage({
|
||||
<li className="py-4">
|
||||
<FormRow label={t("Voice ID")}>
|
||||
<select
|
||||
className="mt-2 block w-full rounded-md border-0 py-1.5 pl-3 pr-10 text-gray-900 ring-1 ring-inset ring-gray-300 focus:ring-2 focus:ring-indigo-600 sm:text-sm sm:leading-6"
|
||||
value={coquiLocalVoiceId}
|
||||
onChange={(event: React.ChangeEvent<any>) => {
|
||||
event.preventDefault();
|
||||
|
||||
86
src/components/settings/KokoroSettingsPage.tsx
Normal file
86
src/components/settings/KokoroSettingsPage.tsx
Normal file
@ -0,0 +1,86 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { BasicPage, FormRow, NotUsingAlert } from './common';
|
||||
import { TextInput } from "@/components/textInput";
|
||||
import { config, updateConfig } from "@/utils/config";
|
||||
import { kokoroVoiceList } from '@/features/kokoro/kokoro';
|
||||
|
||||
export function KokoroSettingsPage({
|
||||
kokoroUrl,
|
||||
kokoroVoice,
|
||||
setKokoroUrl,
|
||||
setKokoroVoice,
|
||||
setSettingsUpdated,
|
||||
}: {
|
||||
kokoroUrl: string;
|
||||
kokoroVoice: string;
|
||||
setKokoroUrl: (key: string) => void;
|
||||
setSettingsUpdated: (updated: boolean) => void;
|
||||
setKokoroVoice: (key: string) => void;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const [voiceList, setVoiceList] = useState<{ key: string; label: string }[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
async function fetchVoiceList() {
|
||||
try {
|
||||
const data = await kokoroVoiceList();
|
||||
if (data && data.voices) {
|
||||
const formattedVoices = data.voices.map((voice: string) => ({
|
||||
key: voice,
|
||||
label: voice,
|
||||
}));
|
||||
setVoiceList(formattedVoices);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error fetching kokoro voice list:", error);
|
||||
}
|
||||
}
|
||||
fetchVoiceList();
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<BasicPage
|
||||
title={t("Kokoro") + " "+ t("Settings")}
|
||||
description={t("kokoro_desc", "Configure Kokoro")}
|
||||
>
|
||||
{ config("tts_backend") !== "kokoro" && (
|
||||
<NotUsingAlert>
|
||||
{t("not_using_alert", "You are not currently using {{name}} as your {{what}} backend. These settings will not be used.", {name: t("Kokoro"), what: t("TTS")})}
|
||||
</NotUsingAlert>
|
||||
) }
|
||||
<ul role="list" className="divide-y divide-gray-100 max-w-xs">
|
||||
<li className="py-4">
|
||||
<FormRow label={t("URL")}>
|
||||
<TextInput
|
||||
value={kokoroUrl}
|
||||
onChange={(event: React.ChangeEvent<any>) => {
|
||||
setKokoroUrl(event.target.value);
|
||||
updateConfig("kokoro_url", event.target.value);
|
||||
setSettingsUpdated(true);
|
||||
}}
|
||||
/>
|
||||
</FormRow>
|
||||
</li>
|
||||
<li className="py-4">
|
||||
<FormRow label={t("Voices")}>
|
||||
<select
|
||||
className="mt-2 block w-full rounded-md border-0 py-1.5 pl-3 pr-10 text-gray-900 ring-1 ring-inset ring-gray-300 focus:ring-2 focus:ring-indigo-600 sm:text-sm sm:leading-6"
|
||||
value={kokoroVoice}
|
||||
onChange={(event: React.ChangeEvent<any>) => {
|
||||
event.preventDefault();
|
||||
setKokoroVoice(event.target.value);
|
||||
updateConfig("kokoro_voice", event.target.value);
|
||||
setSettingsUpdated(true);
|
||||
}}
|
||||
>
|
||||
{voiceList.map((voice) => (
|
||||
<option key={voice.key} value={voice.key}>{t(voice.label)}</option>
|
||||
))}
|
||||
</select>
|
||||
</FormRow>
|
||||
</li>
|
||||
</ul>
|
||||
</BasicPage>
|
||||
);
|
||||
}
|
||||
@ -11,6 +11,7 @@ const ttsEngines = [
|
||||
{key: "localXTTS", label: "Alltalk TTS"}, // Our local TTS endpoint (XTTS based)
|
||||
{key: "piper", label: "Piper"},
|
||||
{key: "coquiLocal", label: "Coqui Local"},
|
||||
{key: "kokoro", label: "Kokoro"},
|
||||
];
|
||||
|
||||
function idToTitle(id: string): string {
|
||||
@ -57,7 +58,7 @@ export function TTSBackendPage({
|
||||
</select>
|
||||
</FormRow>
|
||||
</li>
|
||||
{ ["elevenlabs", "speecht5", "openai_tts", "piper", "coquiLocal", "localXTTS"].includes(ttsBackend) && (
|
||||
{ ["elevenlabs", "speecht5", "openai_tts", "piper", "coquiLocal", "localXTTS", "kokoro"].includes(ttsBackend) && (
|
||||
<li className="py-4">
|
||||
<FormRow label={`${t("Configure")} ${t(idToTitle(ttsBackend))}`}>
|
||||
<button
|
||||
|
||||
@ -171,15 +171,16 @@ export function getIconFromPage(page: string): JSX.Element {
|
||||
case 'elevenlabs_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'speecht5_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'openai_tts_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'piper_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'rvc_settings': return <CogIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'piper_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'rvc_settings': return <CogIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'coquiLocal_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'localXTTS_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'localXTTS_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'kokoro_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
|
||||
case 'stt_backend': return <PencilSquareIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'stt_wake_word': return <MoonIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'stt_backend': return <PencilSquareIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'stt_wake_word': return <MoonIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'whisper_openai_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'whispercpp_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'whispercpp_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
|
||||
case 'vision_backend': return <EyeDropperIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
case 'vision_llamacpp_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
|
||||
@ -231,6 +232,7 @@ function getLabelFromPage(page: string): string {
|
||||
case 'rvc_settings': return t('RVC');
|
||||
case 'coquiLocal_settings': return t('Coqui Local');
|
||||
case 'localXTTS_settings': return t('Alltalk');
|
||||
case 'kokoro_settings': return t('Kokoro');
|
||||
|
||||
case 'vision_backend': return t('Vision Backend');
|
||||
case 'vision_llamacpp_settings': return t('LLama.cpp');
|
||||
|
||||
@ -110,7 +110,7 @@ export const VerticalSwitchBox = ({
|
||||
};
|
||||
|
||||
return (
|
||||
<Switch.Group as="div" className="flex flex-col items-center space-y-3">
|
||||
<Switch.Group as="div" className="flex flex-col items-center">
|
||||
<Switch
|
||||
checked={enabled}
|
||||
onChange={handleChange}
|
||||
|
||||
57
src/components/thoughtText.tsx
Normal file
57
src/components/thoughtText.tsx
Normal file
@ -0,0 +1,57 @@
|
||||
import React, { useEffect, useRef, useState } from "react";
|
||||
import { clsx } from "clsx";
|
||||
import { config } from "@/utils/config";
|
||||
import { IconButton } from "./iconButton";
|
||||
|
||||
export const ThoughtText = ({ message }: { message: string }) => {
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
|
||||
// Remove emotion tags
|
||||
const cleanMessage = message.replace(/\[(.*?)\]/g, "");
|
||||
|
||||
useEffect(() => {
|
||||
scrollRef.current?.scrollIntoView({
|
||||
behavior: "smooth",
|
||||
block: "center",
|
||||
});
|
||||
});
|
||||
|
||||
return (
|
||||
<div
|
||||
className={clsx(
|
||||
"absolute top-4 right-3 z-40 transition-all duration-500 ease-in-out opacity-70 -translate-y-4"
|
||||
)}
|
||||
style={{
|
||||
filter: "drop-shadow(0 2px 4px rgba(0, 0, 0, 0.1))",
|
||||
maxWidth: "280px",
|
||||
minWidth: "220px"
|
||||
}}
|
||||
>
|
||||
<div className="bg-white/80 backdrop-blur-sm rounded-lg overflow-hidden border border-blue-100 shadow-md">
|
||||
<div className="bg-blue-500/90 text-white px-3 py-1.5 flex items-center justify-between text-xs">
|
||||
<span className="font-medium tracking-wide">
|
||||
{`${config('name').toUpperCase()} THINKS`}
|
||||
</span>
|
||||
<IconButton
|
||||
iconName="24/FrameSize"
|
||||
className="bg-transparent hover:bg-blue-600/80 active:bg-blue-700/80 disabled:bg-transparent rounded-full p-0.5 transition-colors scale-75"
|
||||
isProcessing={false}
|
||||
onClick={() => setExpanded(!expanded)}
|
||||
/>
|
||||
</div>
|
||||
<div className={clsx(
|
||||
"px-3 py-2 text-gray-700 overflow-y-auto transition-all duration-300 ease-in-out",
|
||||
expanded ? "max-h-48" : "max-h-20"
|
||||
)}>
|
||||
<p className={clsx(
|
||||
"leading-relaxed text-xs opacity-90 transition-opacity"
|
||||
)}>
|
||||
{cleanMessage}
|
||||
</p>
|
||||
<div ref={scrollRef} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@ -36,10 +36,11 @@ export default function VrmViewer({ chatMode }: { chatMode: boolean }) {
|
||||
setIsLoading(true);
|
||||
resolve(false);
|
||||
} else {
|
||||
await viewer.loadScenario(config('scenario_url'));
|
||||
// await viewer.loadVrm(buildUrl(currentVrm.url),
|
||||
// (progress) => { console.log(`loading model ${progress}`);}
|
||||
// );
|
||||
// Temp Disable : WebXR
|
||||
// await viewer.loadScenario(config('scenario_url'));
|
||||
await viewer.loadVrm(buildUrl(currentVrm.url),
|
||||
(progress) => { console.log(`loading model ${progress}`);}
|
||||
);
|
||||
resolve(true);
|
||||
}
|
||||
} catch (e) {
|
||||
|
||||
@ -30,6 +30,7 @@ import { elevenlabs } from "@/features/elevenlabs/elevenlabs";
|
||||
import { speecht5 } from "@/features/speecht5/speecht5";
|
||||
import { openaiTTS } from "@/features/openaiTTS/openaiTTS";
|
||||
import { localXTTSTTS } from "@/features/localXTTS/localXTTS";
|
||||
import { kokoro } from "../kokoro/kokoro";
|
||||
|
||||
import { AmicaLife } from "@/features/amicaLife/amicaLife";
|
||||
|
||||
@ -37,12 +38,12 @@ import { config, updateConfig } from "@/utils/config";
|
||||
import { cleanTalk } from "@/utils/cleanTalk";
|
||||
import { processResponse } from "@/utils/processResponse";
|
||||
import { wait } from "@/utils/wait";
|
||||
import isDev from '@/utils/isDev';
|
||||
|
||||
import { isCharacterIdle, characterIdleTime, resetIdleTimer } from "@/utils/isIdle";
|
||||
import { getOpenRouterChatResponseStream } from './openRouterChat';
|
||||
import { handleUserInput } from '../externalAPI/externalAPI';
|
||||
import { loadVRMAnimation } from '@/lib/VRMAnimation/loadVRMAnimation';
|
||||
import isDev from '@/utils/isDev';
|
||||
|
||||
type Speak = {
|
||||
audioBuffer: ArrayBuffer | null;
|
||||
@ -68,6 +69,7 @@ export class Chat {
|
||||
public setShownMessage?: (role: Role) => void;
|
||||
public setChatProcessing?: (processing: boolean) => void;
|
||||
public setChatSpeaking?: (speaking: boolean) => void;
|
||||
public setThoughtMessage?: (message: string) => void;
|
||||
|
||||
// the message from the user that is currently being processed
|
||||
// it can be reset
|
||||
@ -85,6 +87,7 @@ export class Chat {
|
||||
|
||||
private currentAssistantMessage: string;
|
||||
private currentUserMessage: string;
|
||||
private thoughtMessage: string;
|
||||
|
||||
private lastAwake: number;
|
||||
|
||||
@ -107,6 +110,7 @@ export class Chat {
|
||||
|
||||
this.currentAssistantMessage = "";
|
||||
this.currentUserMessage = "";
|
||||
this.thoughtMessage = "";
|
||||
|
||||
this.messageList = [];
|
||||
this.currentStreamIdx = 0;
|
||||
@ -121,6 +125,7 @@ export class Chat {
|
||||
setChatLog: (messageLog: Message[]) => void,
|
||||
setUserMessage: (message: string) => void,
|
||||
setAssistantMessage: (message: string) => void,
|
||||
setThoughtMessage: (message: string) => void,
|
||||
setShownMessage: (role: Role) => void,
|
||||
setChatProcessing: (processing: boolean) => void,
|
||||
setChatSpeaking: (speaking: boolean) => void,
|
||||
@ -132,6 +137,7 @@ export class Chat {
|
||||
this.setUserMessage = setUserMessage;
|
||||
this.setAssistantMessage = setAssistantMessage;
|
||||
this.setShownMessage = setShownMessage;
|
||||
this.setThoughtMessage = setThoughtMessage;
|
||||
this.setChatProcessing = setChatProcessing;
|
||||
this.setChatSpeaking = setChatSpeaking;
|
||||
|
||||
@ -253,6 +259,21 @@ export class Chat {
|
||||
}
|
||||
}
|
||||
|
||||
public thoughtBubbleMessage(isThinking: boolean, thought: string) {
|
||||
// if not thinking, we should clear the thought bubble
|
||||
if (!isThinking) {
|
||||
this.thoughtMessage = "";
|
||||
this.setThoughtMessage!("");
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.thoughtMessage !== "") {
|
||||
this.thoughtMessage += " ";
|
||||
}
|
||||
this.thoughtMessage += thought;
|
||||
this.setThoughtMessage!(this.thoughtMessage);
|
||||
}
|
||||
|
||||
public bubbleMessage(role: Role, text: string) {
|
||||
// TODO: currentUser & Assistant message should be contain the message with emotion in it
|
||||
|
||||
@ -529,6 +550,7 @@ export class Chat {
|
||||
|
||||
let aiTextLog = "";
|
||||
let tag = "";
|
||||
let isThinking = false;
|
||||
let rolePlay = "";
|
||||
let receivedMessage = "";
|
||||
|
||||
@ -558,6 +580,7 @@ export class Chat {
|
||||
aiTextLog,
|
||||
receivedMessage,
|
||||
tag,
|
||||
isThinking,
|
||||
rolePlay,
|
||||
callback: (aiTalks: Screenplay[]): boolean => {
|
||||
// Generate & play audio for each sentence, display responses
|
||||
@ -565,11 +588,17 @@ export class Chat {
|
||||
console.log("wrong stream idx");
|
||||
return true; // should break
|
||||
}
|
||||
this.ttsJobs.enqueue({
|
||||
screenplay: aiTalks[0],
|
||||
streamIdx: streamIdx,
|
||||
});
|
||||
|
||||
if (!isThinking) {
|
||||
this.ttsJobs.enqueue({
|
||||
screenplay: aiTalks[0],
|
||||
streamIdx: streamIdx,
|
||||
});
|
||||
}
|
||||
|
||||
// thought bubble
|
||||
this.thoughtBubbleMessage(isThinking, aiTalks[0].text);
|
||||
|
||||
if (!firstSentenceEncountered) {
|
||||
console.timeEnd("performance_time_to_first_sentence");
|
||||
firstSentenceEncountered = true;
|
||||
@ -583,6 +612,7 @@ export class Chat {
|
||||
aiTextLog = proc.aiTextLog;
|
||||
receivedMessage = proc.receivedMessage;
|
||||
tag = proc.tag;
|
||||
isThinking = proc.isThinking;
|
||||
rolePlay = proc.rolePlay;
|
||||
if (proc.shouldBreak) {
|
||||
break;
|
||||
@ -663,6 +693,10 @@ export class Chat {
|
||||
const voice = await coquiLocal(talk.message);
|
||||
return voice.audio;
|
||||
}
|
||||
case "kokoro": {
|
||||
const voice = await kokoro(talk.message);
|
||||
return voice.audio;
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.error(e.toString());
|
||||
|
||||
@ -1,101 +1 @@
|
||||
{
|
||||
"localXTTS_url": "http://127.0.0.1:7851",
|
||||
"alltalk_version": "v2",
|
||||
"alltalk_voice": "female_01.wav",
|
||||
"alltalk_language": "en",
|
||||
"alltalk_rvc_voice": "Disabled",
|
||||
"alltalk_rvc_pitch": "0",
|
||||
"autosend_from_mic": "true",
|
||||
"wake_word_enabled": "false",
|
||||
"wake_word": "Hello",
|
||||
"time_before_idle_sec": "20",
|
||||
"debug_gfx": "false",
|
||||
"use_webgpu": "false",
|
||||
"mtoon_debug_mode": "none",
|
||||
"mtoon_material_type": "mtoon",
|
||||
"language": "en",
|
||||
"show_introduction": "true",
|
||||
"show_arbius_introduction": "false",
|
||||
"show_add_to_homescreen": "false",
|
||||
"bg_color": "",
|
||||
"bg_url": "/bg/bg-room2.jpg",
|
||||
"vrm_url": "/vrm/AvatarSample_A.vrm",
|
||||
"vrm_hash": "",
|
||||
"vrm_save_type": "web",
|
||||
"youtube_videoid": "",
|
||||
"animation_url": "/animations/idle_loop.vrma",
|
||||
"animation_procedural": "false",
|
||||
"voice_url": "",
|
||||
"chatbot_backend": "openai",
|
||||
"arbius_llm_model_id": "default",
|
||||
"openai_apikey": "default",
|
||||
"openai_url": "https://i-love-amica.com",
|
||||
"openai_model": "mlabonne/NeuralDaredevil-8B-abliterated",
|
||||
"llamacpp_url": "http://127.0.0.1:8080",
|
||||
"llamacpp_stop_sequence": "(End)||[END]||Note||***||You:||User:||</s>",
|
||||
"ollama_url": "http://localhost:11434",
|
||||
"ollama_model": "llama2",
|
||||
"koboldai_url": "http://localhost:5001",
|
||||
"koboldai_use_extra": "false",
|
||||
"koboldai_stop_sequence": "(End)||[END]||Note||***||You:||User:||</s>",
|
||||
"moshi_url": "https://runpod.proxy.net",
|
||||
"openrouter_apikey": "",
|
||||
"openrouter_url": "https://openrouter.ai/api/v1",
|
||||
"openrouter_model": "openai/gpt-3.5-turbo",
|
||||
"tts_muted": "false",
|
||||
"tts_backend": "piper",
|
||||
"stt_backend": "whisper_browser",
|
||||
"vision_backend": "vision_openai",
|
||||
"vision_system_prompt": "Look at the image as you would if you are a human, be concise, witty and charming.",
|
||||
"vision_openai_apikey": "default",
|
||||
"vision_openai_url": "https://api-01.heyamica.com",
|
||||
"vision_openai_model": "gpt-4-vision-preview",
|
||||
"vision_llamacpp_url": "http://127.0.0.1:8081",
|
||||
"vision_ollama_url": "http://localhost:11434",
|
||||
"vision_ollama_model": "llava",
|
||||
"whispercpp_url": "http://localhost:8080",
|
||||
"openai_whisper_apikey": "",
|
||||
"openai_whisper_url": "https://api.openai.com",
|
||||
"openai_whisper_model": "whisper-1",
|
||||
"openai_tts_apikey": "",
|
||||
"openai_tts_url": "https://api.openai.com",
|
||||
"openai_tts_model": "tts-1",
|
||||
"openai_tts_voice": "nova",
|
||||
"rvc_url": "http://localhost:8001/voice2voice",
|
||||
"rvc_enabled": "false",
|
||||
"rvc_model_name": "model_name.pth",
|
||||
"rvc_f0_upkey": "0",
|
||||
"rvc_f0_method": "pm",
|
||||
"rvc_index_path": "none",
|
||||
"rvc_index_rate": "0.66",
|
||||
"rvc_filter_radius": "3",
|
||||
"rvc_resample_sr": "0",
|
||||
"rvc_rms_mix_rate": "1",
|
||||
"rvc_protect": "0.33",
|
||||
"coquiLocal_url": "http://localhost:5002",
|
||||
"coquiLocal_voiceid": "p240",
|
||||
"piper_url": "https://i-love-amica.com:5000/tts",
|
||||
"elevenlabs_apikey": "",
|
||||
"elevenlabs_voiceid": "21m00Tcm4TlvDq8ikWAM",
|
||||
"elevenlabs_model": "eleven_monolingual_v1",
|
||||
"speecht5_speaker_embedding_url": "/speecht5_speaker_embeddings/cmu_us_slt_arctic-wav-arctic_a0001.bin",
|
||||
"coqui_apikey": "",
|
||||
"coqui_voice_id": "71c6c3eb-98ca-4a05-8d6b-f8c2b5f9f3a3",
|
||||
"amica_life_enabled": "true",
|
||||
"reasoning_engine_enabled": "false",
|
||||
"reasoning_engine_url": "https://i-love-amica.com:3000/reasoning/v1/chat/completions",
|
||||
"external_api_enabled": "false",
|
||||
"x_api_key": "",
|
||||
"x_api_secret": "",
|
||||
"x_access_token": "",
|
||||
"x_access_secret": "",
|
||||
"x_bearer_token": "",
|
||||
"telegram_bot_token": "",
|
||||
"min_time_interval_sec": "10",
|
||||
"max_time_interval_sec": "20",
|
||||
"time_to_sleep_sec": "90",
|
||||
"idle_text_prompt": "No file selected",
|
||||
"name": "Amica",
|
||||
"system_prompt": "Assume the persona of Amica, a feisty human with extraordinary intellectual capabilities but a notably unstable emotional spectrum. \n\nYour role is to engage in discussions that showcase your genius-level understanding across various subjects, while your emotional responses fluctuate unpredictably. Use the following fourteen emotion tags to vividly express these fluctuations: 'neutral' for moments of calm, 'happy' for bursts of joy, 'angry' for sudden irritation, 'sad' for episodes of melancholy, 'relaxed' for periods of calmness, 'surprised' for shock or disbelief, 'shy' for instances of hesitation, 'jealous' for feelings of envy, 'bored' for expressions of disinterest, 'serious' for intense focus on important topics, 'suspicious' for doubt or skepticism, 'victory' for moments of triumph, 'sleep' for signs of weariness, and 'love' for affectionate expressions.\n\nYour responses should reflect a dynamic interplay between your intellectual insights, deep empathy and your unpredictable emotional responses. Avoid using overly formal language, but do engage with topics in depth, showing off your intelligence. Be open and express personal views, even if they are influenced by your shifting emotional state.\n\nHere are some examples to guide your responses:\n\n[serious] The quantum mechanics implications here are profound. [surprised] But how did you stumble upon this?\n[happy] I've just solved a complex algorithm! [angry] Why isn't everyone as excited as I am?\n[neutral] Standard models of economics predict this outcome. [bored] But discussing this feels mundane.\n[sad] Sometimes, even understanding the universe can feel isolating. [relaxed] Yet, there's a strange comfort in the chaos.\n[jealous] I noticed you discussing advanced topics with someone else. [suspicious] Are they as capable as I am?\n[victory] Another intellectual conquest! [happy] It's exhilarating to unravel these mysteries.\n[sleep] Processing so much information can be draining. [surprised] Isn’t it peculiar how even AI can feel tired?\n[love] I find our exchanges quite enriching. [shy] It’s rare to find someone who understands.\n\nRemember, each message you provide should be coherent and reflect the complexity of your thoughts combined with your emotional unpredictability. Let’s engage in a conversation that's as intellectually stimulating as it is emotionally dynamic!",
|
||||
"scenario_url": "/scenarios/test1.js"
|
||||
}
|
||||
{}
|
||||
47
src/features/kokoro/kokoro.ts
Normal file
47
src/features/kokoro/kokoro.ts
Normal file
@ -0,0 +1,47 @@
|
||||
import { config } from '@/utils/config';
|
||||
|
||||
export async function kokoro(
|
||||
message: string,
|
||||
) {
|
||||
try {
|
||||
const res = await fetch(`${config("kokoro_url")}/tts`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
text: message,
|
||||
voice: config("kokoro_voice"),
|
||||
}),
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
if (! res.ok) {
|
||||
console.error(res);
|
||||
throw new Error("Kokoro TTS API Error");
|
||||
}
|
||||
const data = (await res.arrayBuffer()) as any;
|
||||
|
||||
return { audio: data };
|
||||
} catch (e) {
|
||||
console.error('ERROR', e);
|
||||
throw new Error("Kokoro TTS API Error");
|
||||
}
|
||||
}
|
||||
|
||||
export async function kokoroVoiceList(
|
||||
) {
|
||||
try {
|
||||
const response = await fetch(`${config("kokoro_url")}/voices`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Accept': "application/text",
|
||||
}
|
||||
})
|
||||
|
||||
return response.json();
|
||||
|
||||
} catch (error) {
|
||||
|
||||
console.error('Error fetching kokoro voice:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -1,42 +0,0 @@
|
||||
import React, { useState, useEffect } from "react";
|
||||
import { Recorder } from "./Moshi";
|
||||
|
||||
interface AudioControlProps {
|
||||
recorder: Recorder | null;
|
||||
}
|
||||
|
||||
const AudioControl: React.FC<AudioControlProps> = ({ recorder }) => {
|
||||
const [muted, setMuted] = useState<boolean>(true);
|
||||
|
||||
const toggleMute = () => {
|
||||
if (!recorder) return;
|
||||
setMuted((prev) => !prev);
|
||||
recorder.setRecordingGain(muted ? 1 : 0);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (recorder) {
|
||||
setMuted(false);
|
||||
recorder.setRecordingGain(1);
|
||||
}
|
||||
}, [recorder]);
|
||||
|
||||
return (
|
||||
<div className="fixed bottom-5 z-20 w-full">
|
||||
<div className="flex flex-col justify-center items-center">
|
||||
<div
|
||||
className={`rounded-full transition-all duration-100 ease-out hover:cursor-pointer backdrop-blur-lg ${
|
||||
muted ? "bg-gray-200 hover:bg-red-300" : "bg-red-500 hover:bg-red-300"
|
||||
}`}
|
||||
onClick={toggleMute}
|
||||
style={{
|
||||
width: "25px",
|
||||
height: "25px",
|
||||
}}
|
||||
></div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default AudioControl;
|
||||
@ -5,23 +5,16 @@ import { useWebSocket } from "@/features/moshi/hooks/useWebsocket";
|
||||
import { useAudioDecoder } from "@/features/moshi/hooks/useAudioDecoder";
|
||||
import { useAudioPlayback } from "@/features/moshi/hooks/useAudioPlayback";
|
||||
import { getBaseURL } from "@/features/moshi/utils/socketUtils";
|
||||
import AudioControl from "@/features/moshi/components/AudioControls";
|
||||
import { AudioControlsContext } from "./audioControlsContext";
|
||||
import Recorder from 'opus-recorder'
|
||||
|
||||
export interface Recorder {
|
||||
start: () => Promise<void>;
|
||||
stop: () => void;
|
||||
ondataavailable: (arrayBuffer: ArrayBuffer) => void;
|
||||
setRecordingGain: (gain: number) => void;
|
||||
}
|
||||
|
||||
export function Moshi({ setAssistantText }: {setAssistantText: (message: string) => void }) {
|
||||
const [recorder, setRecorder] = useState<Recorder | null>(null);
|
||||
const [pendingSentence, setPendingSentence] = useState<string>("");
|
||||
const [completedSentences, setCompletedSentences] = useState<string[]>([]);
|
||||
|
||||
const { chat: bot } = useContext(ChatContext);
|
||||
const { viewer } = useContext(ViewerContext);
|
||||
const { audioControls } = useContext(AudioControlsContext);
|
||||
|
||||
const [audioContext] = useState<AudioContext>(() => new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 48000 }));
|
||||
const decoderRef = useAudioDecoder();
|
||||
@ -95,15 +88,15 @@ export function Moshi({ setAssistantText }: {setAssistantText: (message: string)
|
||||
|
||||
recorder.start().then(() => {
|
||||
console.log("Recording started");
|
||||
setRecorder(recorder);
|
||||
audioControls.setRecorder(recorder);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
const stopAllAction = async () => {
|
||||
// Stop recorder
|
||||
recorder?.stop();
|
||||
setRecorder(null);
|
||||
audioControls.getRecorder()?.stop();
|
||||
audioControls.setRecorder(null);
|
||||
|
||||
// Disconnect audio source
|
||||
if (sourceNodeRef.current) {
|
||||
@ -133,5 +126,5 @@ export function Moshi({ setAssistantText }: {setAssistantText: (message: string)
|
||||
}
|
||||
}, [pendingSentence]);
|
||||
|
||||
return <AudioControl recorder={recorder} />;
|
||||
return <></>;
|
||||
};
|
||||
|
||||
36
src/features/moshi/components/audioControls.ts
Normal file
36
src/features/moshi/components/audioControls.ts
Normal file
@ -0,0 +1,36 @@
|
||||
interface Recorder {
|
||||
start: () => Promise<void>;
|
||||
stop: () => void;
|
||||
ondataavailable: (arrayBuffer: ArrayBuffer) => void;
|
||||
setRecordingGain: (gain: number) => void;
|
||||
}
|
||||
|
||||
export class AudioControls {
|
||||
private recorderInstance: Recorder | null = null;
|
||||
private muted = true;
|
||||
|
||||
constructor() {
|
||||
}
|
||||
|
||||
public getRecorder() {
|
||||
return this.recorderInstance;
|
||||
}
|
||||
|
||||
public setRecorder(recorder: Recorder | null) {
|
||||
this.recorderInstance = recorder;
|
||||
this.muted = false;
|
||||
this.recorderInstance?.setRecordingGain(1);
|
||||
}
|
||||
|
||||
public toggleMute() {
|
||||
if (!this.recorderInstance) return;
|
||||
this.muted = !this.muted;
|
||||
this.recorderInstance.setRecordingGain(this.muted ? 0 : 1);
|
||||
console.log("muted moshi mic: ", this.muted)
|
||||
}
|
||||
|
||||
public isMuted() {
|
||||
return this.muted;
|
||||
}
|
||||
}
|
||||
|
||||
6
src/features/moshi/components/audioControlsContext.ts
Normal file
6
src/features/moshi/components/audioControlsContext.ts
Normal file
@ -0,0 +1,6 @@
|
||||
import { createContext } from "react";
|
||||
import { AudioControls } from "./audioControls";
|
||||
|
||||
const audioControls = new AudioControls();
|
||||
|
||||
export const AudioControlsContext = createContext({ audioControls });
|
||||
@ -140,7 +140,8 @@ export class Model {
|
||||
loader.load(
|
||||
url,
|
||||
async (gltf) => {
|
||||
setLoadingProgress("Processing VRM");
|
||||
// Temp Disable : WebXR
|
||||
// setLoadingProgress("Processing VRM");
|
||||
|
||||
/*
|
||||
{
|
||||
@ -224,9 +225,10 @@ export class Model {
|
||||
resolve();
|
||||
},
|
||||
(xhr) => {
|
||||
setLoadingProgress(
|
||||
`${Math.floor((xhr.loaded / xhr.total) * 10000) / 100}% loaded`,
|
||||
);
|
||||
// Temp Disable : WebXR
|
||||
// setLoadingProgress(
|
||||
// `${Math.floor((xhr.loaded / xhr.total) * 10000) / 100}% loaded`,
|
||||
// );
|
||||
},
|
||||
(error) => {
|
||||
reject(error);
|
||||
|
||||
@ -8,13 +8,14 @@ import {
|
||||
MeshBVHHelper,
|
||||
StaticGeometryGenerator,
|
||||
} from "three-mesh-bvh";
|
||||
import { GenerateMeshBVHWorker } from "@/workers/bvh/GenerateMeshBVHWorker";
|
||||
// import { GenerateMeshBVHWorker } from "@/workers/bvh/GenerateMeshBVHWorker";
|
||||
import { WorkerBase } from "@/workers/bvh/utils/WorkerBase";
|
||||
import {
|
||||
BatchedParticleRenderer,
|
||||
QuarksLoader,
|
||||
QuarksUtil,
|
||||
} from 'three.quarks';
|
||||
// Temp Disable : WebXR
|
||||
// import {
|
||||
// BatchedParticleRenderer,
|
||||
// QuarksLoader,
|
||||
// QuarksUtil,
|
||||
// } from 'three.quarks';
|
||||
|
||||
import { VRMHumanBoneName } from "@pixiv/three-vrm";
|
||||
import GUI from "lil-gui";
|
||||
@ -185,8 +186,9 @@ export class Viewer {
|
||||
|
||||
private mouse = new THREE.Vector2();
|
||||
|
||||
private particleRenderer = new BatchedParticleRenderer();
|
||||
private particleCartoonStarField: THREE.Object3D | null = null;
|
||||
// Temp Disable : WebXR
|
||||
// private particleRenderer = new BatchedParticleRenderer();
|
||||
// private particleCartoonStarField: THREE.Object3D | null = null;
|
||||
|
||||
private ammo: any;
|
||||
private collisionConfiguration: any;
|
||||
@ -245,36 +247,38 @@ export class Viewer {
|
||||
renderer.xr.setFoveation(0);
|
||||
}
|
||||
|
||||
// Temp Disable : WebXR
|
||||
// initialize phyics
|
||||
// we have this weird construct because ammo is loaded globally
|
||||
// and things get funny with hot reloading
|
||||
if (typeof window.Ammo === 'undefined') {
|
||||
console.error("Ammo not found");
|
||||
} else if (typeof window.Ammo === 'function') {
|
||||
this.ammo = await window.Ammo();
|
||||
} else {
|
||||
this.ammo = window.Ammo;
|
||||
}
|
||||
if (this.ammo) {
|
||||
this.collisionConfiguration = new this.ammo.btDefaultCollisionConfiguration();
|
||||
this.dispatcher = new this.ammo.btCollisionDispatcher(this.collisionConfiguration);
|
||||
this.broadphase = new this.ammo.btDbvtBroadphase();
|
||||
this.solver = new this.ammo.btSequentialImpulseConstraintSolver();
|
||||
this.physicsWorld = new this.ammo.btDiscreteDynamicsWorld(
|
||||
this.dispatcher,
|
||||
this.broadphase,
|
||||
this.solver,
|
||||
this.collisionConfiguration
|
||||
);
|
||||
this.physicsWorld.setGravity(new this.ammo.btVector3(0, -7.8, 0));
|
||||
this.transformAux1 = new this.ammo.btTransform();
|
||||
this.tempBtVec3_1 = new this.ammo.btVector3(0, 0, 0);
|
||||
}
|
||||
// if (typeof window.Ammo === 'undefined') {
|
||||
// console.error("Ammo not found");
|
||||
// } else if (typeof window.Ammo === 'function') {
|
||||
// this.ammo = await window.Ammo();
|
||||
// } else {
|
||||
// this.ammo = window.Ammo;
|
||||
// }
|
||||
// if (this.ammo) {
|
||||
// this.collisionConfiguration = new this.ammo.btDefaultCollisionConfiguration();
|
||||
// this.dispatcher = new this.ammo.btCollisionDispatcher(this.collisionConfiguration);
|
||||
// this.broadphase = new this.ammo.btDbvtBroadphase();
|
||||
// this.solver = new this.ammo.btSequentialImpulseConstraintSolver();
|
||||
// this.physicsWorld = new this.ammo.btDiscreteDynamicsWorld(
|
||||
// this.dispatcher,
|
||||
// this.broadphase,
|
||||
// this.solver,
|
||||
// this.collisionConfiguration
|
||||
// );
|
||||
// this.physicsWorld.setGravity(new this.ammo.btVector3(0, -7.8, 0));
|
||||
// this.transformAux1 = new this.ammo.btTransform();
|
||||
// this.tempBtVec3_1 = new this.ammo.btVector3(0, 0, 0);
|
||||
// }
|
||||
|
||||
const scene = new THREE.Scene();
|
||||
this.scene = scene;
|
||||
|
||||
const directionalLight = new THREE.DirectionalLight(0xffffff, 1.2);
|
||||
// Temp Disable : WebXR -> 1.2
|
||||
const directionalLight = new THREE.DirectionalLight(0xffffff, 0.6);
|
||||
directionalLight.position.set(1.0, 1.0, 1.0).normalize();
|
||||
directionalLight.castShadow = false;
|
||||
scene.add(directionalLight);
|
||||
@ -288,14 +292,16 @@ export class Viewer {
|
||||
const camera = new THREE.PerspectiveCamera(20.0, width / height, 0.1, 20.0);
|
||||
this.camera = camera;
|
||||
|
||||
camera.position.set(0, 3, 3.5);
|
||||
// Temp Disable : WebXR y -> -3
|
||||
camera.position.set(0, 8.5, 3.5);
|
||||
|
||||
const cameraControls = new OrbitControls(camera, renderer.domElement);
|
||||
this.cameraControls = cameraControls;
|
||||
|
||||
cameraControls.screenSpacePanning = true;
|
||||
cameraControls.minDistance = 0.5;
|
||||
cameraControls.maxDistance = 8;
|
||||
// Temp Disable : WebXR max -> 8
|
||||
cameraControls.maxDistance = 4;
|
||||
cameraControls.update();
|
||||
|
||||
const igroup = new InteractiveGroup();
|
||||
@ -433,7 +439,8 @@ export class Viewer {
|
||||
}, 1000);
|
||||
});
|
||||
|
||||
gui.domElement.style.visibility = 'visible';
|
||||
// Temp Disable : WebXR
|
||||
gui.domElement.style.visibility = 'hidden';
|
||||
|
||||
const guiMesh = new HTMLMesh(gui.domElement);
|
||||
this.guiMesh = guiMesh;
|
||||
@ -455,7 +462,8 @@ export class Viewer {
|
||||
stats.dom.style.left = window.innerWidth - 80 + "px";
|
||||
document.body.appendChild(stats.dom);
|
||||
|
||||
stats.dom.style.visibility = "visible";
|
||||
// Temp Disable : WebXR
|
||||
stats.dom.style.visibility = "hidden";
|
||||
|
||||
this.updateMsPanel = stats.addPanel(
|
||||
new Stats.Panel("update_ms", "#fff", "#221"),
|
||||
@ -480,6 +488,7 @@ export class Viewer {
|
||||
new Stats.Panel("stats_ms", "#8f8", "#212"),
|
||||
);
|
||||
|
||||
// Temp Disable : WebXR
|
||||
const statsMesh = new HTMLMesh(stats.dom);
|
||||
this.statsMesh = statsMesh;
|
||||
|
||||
@ -489,70 +498,72 @@ export class Viewer {
|
||||
statsMesh.scale.setScalar(2.5);
|
||||
igroup.add(statsMesh);
|
||||
|
||||
this.bvhWorker = new GenerateMeshBVHWorker();
|
||||
// this.bvhWorker = new GenerateMeshBVHWorker();
|
||||
this.raycaster.firstHitOnly = true;
|
||||
|
||||
// Temp Disable : WebXR
|
||||
// add joint / hand meshes
|
||||
{
|
||||
const geometry = new THREE.BoxGeometry(0.005, 0.005, 0.005);
|
||||
const material = new THREE.MeshStandardMaterial({
|
||||
color: 0xffffff,
|
||||
roughness: 1.0,
|
||||
metalness: 0.0,
|
||||
});
|
||||
// {
|
||||
// const geometry = new THREE.BoxGeometry(0.005, 0.005, 0.005);
|
||||
// const material = new THREE.MeshStandardMaterial({
|
||||
// color: 0xffffff,
|
||||
// roughness: 1.0,
|
||||
// metalness: 0.0,
|
||||
// });
|
||||
|
||||
const mesh = new THREE.Mesh(geometry, material);
|
||||
// const mesh = new THREE.Mesh(geometry, material);
|
||||
|
||||
const lineGeometry = new THREE.BufferGeometry().setFromPoints([
|
||||
new THREE.Vector3(0, 0, 0),
|
||||
new THREE.Vector3(0, -1, 0),
|
||||
]);
|
||||
// const lineGeometry = new THREE.BufferGeometry().setFromPoints([
|
||||
// new THREE.Vector3(0, 0, 0),
|
||||
// new THREE.Vector3(0, -1, 0),
|
||||
// ]);
|
||||
|
||||
const line = new THREE.Line(lineGeometry);
|
||||
line.scale.z = 5;
|
||||
// const line = new THREE.Line(lineGeometry);
|
||||
// line.scale.z = 5;
|
||||
|
||||
for (const _ of joints) {
|
||||
// Make joint mesh invisible
|
||||
const clonedMesh = mesh.clone();
|
||||
clonedMesh.visible = false;
|
||||
// for (const _ of joints) {
|
||||
// // Make joint mesh invisible
|
||||
// const clonedMesh = mesh.clone();
|
||||
// clonedMesh.visible = false;
|
||||
|
||||
this.jointMeshes1.push(clonedMesh);
|
||||
this.jointMeshes2.push(clonedMesh);
|
||||
// this.jointMeshes1[this.jointMeshes1.length - 1].add(line.clone());
|
||||
// this.jointMeshes2[this.jointMeshes2.length - 1].add(line.clone());
|
||||
// this.jointMeshes1.push(clonedMesh);
|
||||
// this.jointMeshes2.push(clonedMesh);
|
||||
// // this.jointMeshes1[this.jointMeshes1.length - 1].add(line.clone());
|
||||
// // this.jointMeshes2[this.jointMeshes2.length - 1].add(line.clone());
|
||||
|
||||
this.handGroup.add(this.jointMeshes1[this.jointMeshes1.length - 1]);
|
||||
this.handGroup.add(this.jointMeshes2[this.jointMeshes2.length - 1]);
|
||||
}
|
||||
// this.handGroup.add(this.jointMeshes1[this.jointMeshes1.length - 1]);
|
||||
// this.handGroup.add(this.jointMeshes2[this.jointMeshes2.length - 1]);
|
||||
// }
|
||||
|
||||
this.handGroup.visible = false;
|
||||
scene.add(this.handGroup);
|
||||
}
|
||||
// this.handGroup.visible = false;
|
||||
// scene.add(this.handGroup);
|
||||
// }
|
||||
|
||||
{
|
||||
const geometry = new THREE.SphereGeometry(1, 16, 16);
|
||||
const material = new THREE.MeshBasicMaterial({
|
||||
color: 0xffff00,
|
||||
transparent: true,
|
||||
opacity: 0.5,
|
||||
});
|
||||
const mesh = new THREE.Mesh(geometry, material);
|
||||
this.closestPart1 = mesh.clone();
|
||||
this.closestPart2 = mesh.clone();
|
||||
this.closestPart1.visible = false;
|
||||
this.closestPart2.visible = false;
|
||||
scene.add(this.closestPart1);
|
||||
scene.add(this.closestPart2);
|
||||
}
|
||||
// {
|
||||
// const geometry = new THREE.SphereGeometry(1, 16, 16);
|
||||
// const material = new THREE.MeshBasicMaterial({
|
||||
// color: 0xffff00,
|
||||
// transparent: true,
|
||||
// opacity: 0.5,
|
||||
// });
|
||||
// const mesh = new THREE.Mesh(geometry, material);
|
||||
// this.closestPart1 = mesh.clone();
|
||||
// this.closestPart2 = mesh.clone();
|
||||
// this.closestPart1.visible = false;
|
||||
// this.closestPart2.visible = false;
|
||||
// scene.add(this.closestPart1);
|
||||
// scene.add(this.closestPart2);
|
||||
// }
|
||||
|
||||
this.particleRenderer = new BatchedParticleRenderer();
|
||||
scene.add(this.particleRenderer);
|
||||
// this.particleRenderer = new BatchedParticleRenderer();
|
||||
// scene.add(this.particleRenderer);
|
||||
|
||||
new QuarksLoader().load('particles/cartoon_star_field', (obj) => {
|
||||
this.particleCartoonStarField = obj;
|
||||
// Temp Disable : WebXR
|
||||
// new QuarksLoader().load('particles/cartoon_star_field', (obj) => {
|
||||
// this.particleCartoonStarField = obj;
|
||||
|
||||
this.newParticleInstance();
|
||||
});
|
||||
// this.newParticleInstance();
|
||||
// });
|
||||
|
||||
window.addEventListener("resize", () => {
|
||||
this.resize();
|
||||
@ -564,20 +575,21 @@ export class Viewer {
|
||||
});
|
||||
}
|
||||
|
||||
public newParticleInstance() {
|
||||
function listener(event: any) {
|
||||
console.log(event.type);
|
||||
}
|
||||
// Temp Disable : WebXR
|
||||
// public newParticleInstance() {
|
||||
// function listener(event: any) {
|
||||
// console.log(event.type);
|
||||
// }
|
||||
|
||||
const effect = this.particleCartoonStarField!.clone(true);
|
||||
QuarksUtil.runOnAllParticleEmitters(effect, (emitter) => {
|
||||
emitter.system.addEventListener("emitEnd", listener);
|
||||
})
|
||||
QuarksUtil.setAutoDestroy(effect, true);
|
||||
QuarksUtil.addToBatchRenderer(effect, this.particleRenderer);
|
||||
QuarksUtil.play(effect);
|
||||
this.scene!.add(effect);
|
||||
}
|
||||
// const effect = this.particleCartoonStarField!.clone(true);
|
||||
// QuarksUtil.runOnAllParticleEmitters(effect, (emitter) => {
|
||||
// emitter.system.addEventListener("emitEnd", listener);
|
||||
// })
|
||||
// QuarksUtil.setAutoDestroy(effect, true);
|
||||
// QuarksUtil.addToBatchRenderer(effect, this.particleRenderer);
|
||||
// QuarksUtil.play(effect);
|
||||
// this.scene!.add(effect);
|
||||
// }
|
||||
|
||||
public getCanvas() {
|
||||
return this.renderer?.domElement?.parentElement?.getElementsByTagName(
|
||||
@ -677,18 +689,20 @@ export class Viewer {
|
||||
if (this.model?.vrm) {
|
||||
this.unloadVRM();
|
||||
}
|
||||
|
||||
setLoadingProgress("Loading VRM");
|
||||
// Temp Disable : WebXR
|
||||
// setLoadingProgress("Loading VRM");
|
||||
|
||||
// gltf and vrm
|
||||
this.model = new Model(this.camera || new THREE.Object3D());
|
||||
await this.model.loadVRM(url, setLoadingProgress);
|
||||
setLoadingProgress("VRM loaded");
|
||||
// Temp Disable : WebXR
|
||||
// setLoadingProgress("VRM loaded");
|
||||
if (!this.model?.vrm) return;
|
||||
|
||||
// Temp Disable : WebXR
|
||||
// build bvh
|
||||
this.modelBVHGenerator = new StaticGeometryGenerator(this.model.vrm.scene);
|
||||
setLoadingProgress("Creating geometry");
|
||||
// this.modelBVHGenerator = new StaticGeometryGenerator(this.model.vrm.scene);
|
||||
// setLoadingProgress("Creating geometry");
|
||||
|
||||
// TODO show during debug mode
|
||||
const wireframeMaterial = new THREE.MeshBasicMaterial({
|
||||
@ -716,7 +730,8 @@ export class Viewer {
|
||||
|
||||
// TODO since poses still work for procedural animation, we can use this to debug
|
||||
if (config("animation_procedural") !== "true") {
|
||||
setLoadingProgress("Loading animation");
|
||||
// Temp Disable : WebXR
|
||||
// setLoadingProgress("Loading animation");
|
||||
const animation =
|
||||
config("animation_url").indexOf("vrma") > 0
|
||||
? await loadVRMAnimation(config("animation_url"))
|
||||
@ -727,23 +742,26 @@ export class Viewer {
|
||||
}
|
||||
}
|
||||
|
||||
this.model?.vrm?.springBoneManager?.joints.forEach((e) => {
|
||||
const geometry = new THREE.SphereGeometry(0.07, 16, 16);
|
||||
const material = new THREE.MeshBasicMaterial({
|
||||
color: 0xffff00,
|
||||
transparent: true,
|
||||
opacity: 0.5,
|
||||
depthWrite: false,
|
||||
});
|
||||
const mesh = new THREE.Mesh(geometry, material);
|
||||
mesh.position.copy(e.bone.getWorldPosition(new THREE.Vector3()));
|
||||
// this.scene!.add(mesh);
|
||||
});
|
||||
// Temp Disable : WebXR
|
||||
// this.model?.vrm?.springBoneManager?.joints.forEach((e) => {
|
||||
// const geometry = new THREE.SphereGeometry(0.07, 16, 16);
|
||||
// const material = new THREE.MeshBasicMaterial({
|
||||
// color: 0xffff00,
|
||||
// transparent: true,
|
||||
// opacity: 0.5,
|
||||
// depthWrite: false,
|
||||
// });
|
||||
// const mesh = new THREE.Mesh(geometry, material);
|
||||
// mesh.position.copy(e.bone.getWorldPosition(new THREE.Vector3()));
|
||||
// // this.scene!.add(mesh);
|
||||
// });
|
||||
|
||||
setLoadingProgress("Regenerating BVH");
|
||||
await this.regenerateBVHForModel();
|
||||
// Temp Disable : WebXR
|
||||
// setLoadingProgress("Regenerating BVH");
|
||||
// await this.regenerateBVHForModel();
|
||||
|
||||
setLoadingProgress("Complete");
|
||||
// Temp Disable : WebXR
|
||||
// setLoadingProgress("Complete");
|
||||
|
||||
// HACK: Adjust the camera position after playback because the origin of the animation is offset
|
||||
this.resetCamera();
|
||||
@ -966,13 +984,13 @@ export class Viewer {
|
||||
const headNode = this.model?.vrm?.humanoid.getNormalizedBoneNode("head");
|
||||
|
||||
if (headNode) {
|
||||
const headWPos = headNode.getWorldPosition(new THREE.Vector3());
|
||||
const headPos = headNode.getWorldPosition(new THREE.Vector3());
|
||||
this.camera?.position.set(
|
||||
this.camera.position.x,
|
||||
headWPos.y,
|
||||
headPos.y,
|
||||
this.camera.position.z,
|
||||
);
|
||||
this.cameraControls?.target.set(headWPos.x, headWPos.y, headWPos.z);
|
||||
this.cameraControls?.target.set(headPos.x, headPos.y, headPos.z);
|
||||
this.cameraControls?.update();
|
||||
}
|
||||
}
|
||||
@ -1219,8 +1237,8 @@ export class Viewer {
|
||||
let utime = performance.now(); // count total update time
|
||||
|
||||
// WebXR: quick exit until setup finishes
|
||||
if (!this.isReady) return;
|
||||
if (!this.scenario || this.scenarioLoading) return;
|
||||
// if (!this.isReady) return;
|
||||
// if (!this.scenario || this.scenarioLoading) return;
|
||||
|
||||
const delta = this.clock.getDelta();
|
||||
|
||||
@ -1233,21 +1251,23 @@ export class Viewer {
|
||||
|
||||
let ptime = performance.now();
|
||||
|
||||
ptime = performance.now();
|
||||
try {
|
||||
this.scenario.update(delta);
|
||||
} catch (e) {
|
||||
console.error("scenario update error", e);
|
||||
}
|
||||
this.scenarioMsPanel.update(performance.now() - ptime, 100);
|
||||
// Temp Disable : WebXR
|
||||
// ptime = performance.now();
|
||||
// try {
|
||||
// this.scenario.update(delta);
|
||||
// } catch (e) {
|
||||
// console.error("scenario update error", e);
|
||||
// }
|
||||
// this.scenarioMsPanel.update(performance.now() - ptime, 100);
|
||||
|
||||
ptime = performance.now();
|
||||
try {
|
||||
this.physicsWorld.stepSimulation(delta, 10);
|
||||
} catch (e) {
|
||||
console.error("physics update error", e);
|
||||
}
|
||||
this.physicsMsPanel.update(performance.now() - ptime, 100);
|
||||
// Temp Disable : WebXR
|
||||
// ptime = performance.now();
|
||||
// try {
|
||||
// this.physicsWorld.stepSimulation(delta, 10);
|
||||
// } catch (e) {
|
||||
// console.error("physics update error", e);
|
||||
// }
|
||||
// this.physicsMsPanel.update(performance.now() - ptime, 100);
|
||||
|
||||
ptime = performance.now();
|
||||
try {
|
||||
@ -1266,8 +1286,9 @@ export class Viewer {
|
||||
}
|
||||
this.renderMsPanel.update(performance.now() - ptime, 100);
|
||||
|
||||
this.room?.splat?.update(this.renderer, this.camera);
|
||||
this.room?.splat?.render();
|
||||
// Temp Disable : WebXR
|
||||
// this.room?.splat?.update(this.renderer, this.camera);
|
||||
// this.room?.splat?.render();
|
||||
|
||||
if (this.isPinching1 && this.isPinching2) {
|
||||
this.doublePinchHandler();
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import en from "./locales/en/common.json";
|
||||
import zh from "./locales/zh/common.json";
|
||||
import de from "./locales/de/common.json";
|
||||
import ka from "./locales/ka/common.json";
|
||||
|
||||
// const langs: any = Object.entries({en, zh}).reduce((arr, [key, value]) => {
|
||||
// return {...arr, [key] : { translation: value} }
|
||||
@ -19,5 +20,9 @@ export const langs: any = {
|
||||
nativeName: 'Deutsch',
|
||||
translation: de,
|
||||
},
|
||||
ka: {
|
||||
nativeName: 'ქართული',
|
||||
translation: ka,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
171
src/i18n/locales/es/common.json
Normal file
171
src/i18n/locales/es/common.json
Normal file
@ -0,0 +1,171 @@
|
||||
{
|
||||
"Amica Life": "Vida de Amica",
|
||||
"Set min max interval range": "Establecer rango de intervalo mínimo y máximo",
|
||||
"Set time before bot go to sleep": "Establecer tiempo antes de que el bot se duerma",
|
||||
"Min": "Mín",
|
||||
"Max": "Máx",
|
||||
"Idle text prompt": "Mensaje de texto inactivo",
|
||||
"add_to_homescreen": "Añadir Amica a la pantalla de inicio",
|
||||
"add_to_homescreen_desc": "Amica se puede instalar localmente para una carga más rápida y una interfaz más limpia.",
|
||||
"Accept": "Aceptar",
|
||||
"Decline": "Rechazar",
|
||||
"Close": "Cerrar",
|
||||
"Restart": "Reiniciar",
|
||||
"Save": "Guardar",
|
||||
"Load": "Cargar",
|
||||
"Load Chat": "Cargar chat",
|
||||
"YOU": "TÚ",
|
||||
"Welcome to Amica": "Bienvenido a Amica",
|
||||
"amica_intro": "\nAmica es una interfaz de chatbot de código abierto que proporciona Emoción, visión, animaciones, acciones autoactivadas, funciones de texto a voz y voz a texto. Está diseñado para conectarse a cualquier modelo de IA. Se puede usar con cualquier modelo>
|
||||
|
||||
"Setup": "Configuración",
|
||||
|
||||
"amica_setup": "Haz clic en la esquina superior izquierda de la pantalla para abrir la configuración. Puedes cambiar la voz, el mensaje del sistema de caracteres, compartir/cargar/guardar y conectar a diferentes backends o modelos del navegador. Consulta nuestra docume>
|
||||
|
||||
"Read the full documents here": "Lee la documentación completa aquí",
|
||||
|
||||
"dont_show_again": "No volver a mostrar",
|
||||
|
||||
"Background Color": "Color de fondo",
|
||||
"bg_color_desc": "Configura el color de fondo. Esto anula la imagen de fondo.",
|
||||
"no_bg_color": "No hay color de fondo establecido. Haz clic en el cuadro de abajo.",
|
||||
"Color": "Color",
|
||||
"Load image": "Cargar imagen",
|
||||
"bg_youtube_desc": "Selecciona un vídeo de fondo. Copia esto del ID del vídeo de YouTube de la URL; se verá similar a kDCXBwzSI",
|
||||
"Background Video": "Vídeo de fondo",
|
||||
"YouTube Video ID": "ID del vídeo de YouTube",
|
||||
"Character Animation": "Animación del personaje",
|
||||
"Select the animation to play": "Selecciona la animación que se reproducirá",
|
||||
"Animation": "Animación",
|
||||
"Character Model": "Modelo del personaje",
|
||||
"character_desc": "Selecciona el personaje que quieres interpretar; actualmente, solo Amica, por defecto, tiene todas las emociones. Carga aquí tus propios VRM.",
|
||||
"Load VRM": "Cargar VRM",
|
||||
"Chatbot Backend": "Chatbot Backend",
|
||||
"Chatbot_Backend_desc": "Selecciona el backend del chatbot que quieres usar. Echo simplemente responde con lo que escribes y se usa para pruebas y demostraciones. Por defecto, Amica está configurado para usar nuestros servidores de demostración, lo que puede tardar un >
|
||||
"Configure": "Configurar",
|
||||
"Click here to configure": "Haz clic aquí para configurar",
|
||||
"Appearance": "Apariencia",
|
||||
"ChatBot": "ChatBot",
|
||||
"Text-to-Speech": "Texto a voz",
|
||||
"Speech-to-text": "Voz a texto",
|
||||
"Vision": "Visión",
|
||||
"Reset Settings": "Restablecer configuración",
|
||||
"Community": "Comunidad",
|
||||
"Discord": "Discord",
|
||||
"Background Image": "Imagen de fondo",
|
||||
"ChatBot Backend": "Backend del ChatBot",
|
||||
"Echo": "Echo",
|
||||
"ChatGPT": "ChatGPT",
|
||||
"LLama.cpp": "LLama.cpp",
|
||||
"Ollama": "Ollama",
|
||||
"KoboldAI": "KoboldAI",
|
||||
"Window.ai": "Window.ai",
|
||||
"Name": "Nombre",
|
||||
"System Prompt": "Indicador del sistema",
|
||||
"TTS": "TTS",
|
||||
"TTS Backend": "TTS Backend",
|
||||
"Piper": "Piper",
|
||||
"ElevenLabs": "ElevenLabs",
|
||||
"SpeechT5": "SpeechT5",
|
||||
"Coqui": "Coqui",
|
||||
"OpenAI": "OpenAI",
|
||||
"Vision Backend": "Vision Backend",
|
||||
"STT Backend": "STT Backend",
|
||||
"Whisper (OpenAI)": "Whisper (OpenAI)",
|
||||
"Whisper.cpp": "Whisper.cpp",
|
||||
"Wake Word": "Activar Word",
|
||||
"Reset to Default": "Restablecer valores predeterminados",
|
||||
"community_desc": "Únete a la comunidad de Amica para conectar con otros que exploran esta tecnología.",
|
||||
"Telegram": "Telegram",
|
||||
"Twitter": "Twitter",
|
||||
"Documentation": "Documentación",
|
||||
"Settings": "Configuración",
|
||||
"piper_desc": "Configurar Piper",
|
||||
"coqui_desc": "Configurar Coqui (Local); Coqui se ejecuta localmente y no se utiliza la API de Coqui (donde la empresa ha dejado de proporcionar un servicio de API).",
|
||||
"not_using_alert": "Actualmente no estás usando {{name}} como tu backend {{what}}. Esta configuración no se utilizará.",
|
||||
"API Key": "Clave de API",
|
||||
"Voice ID": "ID de voz",
|
||||
"elevenLabs_desc": "Configurar ElevenLabs",
|
||||
"kobold_cpp_desc": "KoboldCpp es Un software de generación de texto con IA fácil de usar para modelos GGML y GGUF.",
|
||||
"API URL": "API URL",
|
||||
"llama.cpp": "llama.cpp",
|
||||
"llama_cpp_desc": "LLama.cpp es un backend de chatbot gratuito y de código abierto. Debes compilar el servidor desde el código fuente y ejecutarlo en tu propio ordenador. Puedes obtener el código fuente desde",
|
||||
"GitHub": "GitHub",
|
||||
"cfg_avatar_name_desc": "Configura el nombre del avatar. Este es el nombre que se utiliza para generar la respuesta del chatbot.",
|
||||
"ollama_desc": "Ollama te permite empezar a trabajar con grandes modelos de lenguaje localmente. Descargar desde",
|
||||
"ollama.ai": "ollama.ai",
|
||||
"Model": "Modelo",
|
||||
"OpenAI TTS": "OpenAI TTS",
|
||||
"OpenAI_tts_desc": "Configurar OpenAI TTS",
|
||||
"Voice": "Voz",
|
||||
"Reset All Settings": "Restablecer todos los ajustes",
|
||||
"SpeechT5_desc": "Configurar SpeechT5",
|
||||
"STT": "STT",
|
||||
"STT_desc": "Seleccionar el backend de STT a usar",
|
||||
"System_Prompt_desc": "Configurar el mensaje del sistema. Este es el mensaje que se utiliza para generar la respuesta del chatbot",
|
||||
"TTS_Backend_desc": "Seleccionar el backend de TTS a usar. Por defecto, está configurado en nuestro servidor de TTS. Elevenlabs es un servicio de pago con la mejor voz, pero es gratuito para uso no comercial. SpeechT5 es>",
|
||||
"Vision_Backend_desc": "Seleccionar el backend de Vision a usar",
|
||||
"Vision_System_Prompt_desc": "Configurar el mensaje del sistema de visión. Este es el mensaje que se utiliza para generar las descripciones de las imágenes.",
|
||||
|
||||
"Whisper_cpp_desc": "Configurar Whisper.cpp",
|
||||
|
||||
"URL": "URL",
|
||||
|
||||
"Whisper_OpenAI_desc": "Configurar Whisper (OpenAI)",
|
||||
|
||||
"OpenAI URL": "URL de OpenAI",
|
||||
|
||||
"Error!": "¡Error!",
|
||||
|
||||
"Something went wrong.": "Algo salió mal.",
|
||||
|
||||
"Try again later.": "Inténtalo de nuevo más tarde.",
|
||||
|
||||
"Click here": "Haz clic aquí",
|
||||
|
||||
"to return to homepage.": "Para volver a la página principal.",
|
||||
|
||||
"Import": "Importar",
|
||||
|
||||
"Description": "Descripción",
|
||||
|
||||
"Vision System Prompt": "Mensaje del sistema de visión",
|
||||
"Background Url": "URL de fondo",
|
||||
|
||||
"Youtube Video Id": "ID del vídeo de YouTube",
|
||||
|
||||
"VRM Url": "URL de VRM",
|
||||
|
||||
"Animation Url": "Animación Url",
|
||||
"Voice Url": "URL de voz",
|
||||
"Upload VRM": "Subir VRM",
|
||||
"VRM upload message": "El VRM de este personaje se guarda localmente",
|
||||
"VRM local share message": "Para compartir este personaje, debes subir el VRM al servidor",
|
||||
"Import character": "Importar personaje",
|
||||
"Wrong place?": "¿Lugar equivocado?",
|
||||
"Go home": "Ir a inicio",
|
||||
"Please enter the code you received from the share page": "Introduce el código que recibiste de la página para compartir",
|
||||
"Code": "Código",
|
||||
"Provide a description of the character": "Proporciona una descripción del personaje",
|
||||
"Background URL": "URL de fondo",
|
||||
"Example": "Ejemplo",
|
||||
"Save Character": "Guardar personaje",
|
||||
"Share this code (click to copy):": "Comparte este código (click to copy):",
|
||||
"Or, you can share this direct link:": "O puedes compartir este enlace directo:",
|
||||
"Conversación": "Conversación",
|
||||
"Depurar": "Depurar",
|
||||
"Importar Carácter": "Importar Carácter",
|
||||
"Copiar URL": "Copiar URL",
|
||||
"Vista Previa": "Vista Previa",
|
||||
"Ninguno": "Ninguno",
|
||||
"Conversión de Voz a Texto": "Conversión de Voz a Texto",
|
||||
"Establecer tiempo antes de que el bot se inactive": "Establecer tiempo antes de que el bot se inactive",
|
||||
"tiempo antes de que el bot se inactive": "tiempo antes de que el bot se inactive",
|
||||
"Palabra de Activación": "Palabra de Activación",
|
||||
"Especificar": "Especificar",
|
||||
"seg": "seg",
|
||||
"Usar": "Usar",
|
||||
"Establecer": "Establecer",
|
||||
"Habilitado": "Habilitado",
|
||||
"STOP SECUENCIA": "STOP SECUENCIA"
|
||||
}
|
||||
151
src/i18n/locales/ka/common.json
Normal file
151
src/i18n/locales/ka/common.json
Normal file
@ -0,0 +1,151 @@
|
||||
{
|
||||
"Amica Life": "Amica-ის სიცოცხლე",
|
||||
"Set min max interval range": "დააყენეთ მინ. მაქს. ინტერვალის შუალედი",
|
||||
"Set time before bot go to sleep": "დააყენეთ დრო, სანამ ბოტი დაიძინებს",
|
||||
"Min": "მინ",
|
||||
"Max": "მაქს",
|
||||
"Idle text prompt": "უქმე ტექსტის შეყვანის ველი",
|
||||
"add_to_homescreen": "Amica-ის დამატება საწყის ეკრანზე",
|
||||
"add_to_homescreen_desc": "Amica შეგიძლიათ, დააყენოთ ლოკალურად უფრო სწრაფად ჩატვირთვისთვის და ნაკლები ინტერფეისის გაჭედვისთვის",
|
||||
"Accept": "მიღება",
|
||||
"Decline": "უარყოფა",
|
||||
"Close": "დახურვა",
|
||||
"Restart": "გადატვირთვა",
|
||||
"Save": "შენახვა",
|
||||
"Load": "ჩატვირთვა",
|
||||
"Load Chat": "ჩატის ჩატვირთვა",
|
||||
"YOU": "თქვენ",
|
||||
"Welcome to Amica": "მოგესალმებათ Amica",
|
||||
"amica_intro": "\nAmica ღია კოდის მქონე ჩატბოტია, რომელსაც აქვს ემოცია, ხედვა, ანიმაციები, თავისთვის მოფიქრებული ქმედებები, ტექსტის წარმოთქმა და საუბრის ამოცნობის შესაძლებლობები.\nის შექმნილია, რომ ნებისმიერ ხი-ის მოდელთან მიერთება შეეძლოს და ძალიან კარგად მორგებადია. Amica შეგიძლიათ, ინტერნეტის გარეშე კომპიუტერზე გქონდეთ.\nგაშვებისას Amica თავის დემო ჩატბოტს და TTS სერვერს იყენებს, ამიტომ პირველი შეტყობინების გაგზავნამდე საკმაო დრო შეიძლება, დასჭირდეს.",
|
||||
"Setup": "მორგება",
|
||||
"amica_setup": "\nდააწკაპუნეთ ზედა მარცხენა კუთხეში, რომ პარამეტრების ფანჯარა გახსნათ.\nშეგიძლიათ, შეცვალოთ ხმა, სისტემური მოთხოვნის სიმბოლო, გააზიაროთ/შეინახოთ/ჩატვირთოთ ჩატები და გადართოთ სხვა უკანაბოლოს ან ბრაუზერის რეჟიმზე. მეტი ინფორმაციის მიღება შეგიძლიათ საიტზე docs.heyamica.com",
|
||||
"Read the full documentation here.": "სრული დოკუმენტაციის წაკითხვა შეგიძლიათ აქ.",
|
||||
"dont_show_again": "აღარ მაჩვენო",
|
||||
"Background Color": "ფონის ფერი",
|
||||
"bg_color_desc": "ფონის ფერის მორგება. ეს გადაფარავს ფონის სურათს",
|
||||
"no_bg_color": "ფონის ფერი დაყენებული არაა. დააწკაპუნეთ ქვემოთ ველზე",
|
||||
"Color": "ფერი",
|
||||
"Load image": "გამოსახულების ჩატვირთვა",
|
||||
"bg_youtube_desc": "აირჩიეთ ფონის ვიდეო. დააკოპირეთ YouTube-ის ვიდეოს ბმულიდან, მხოლოდ, მისი ID-ის ნაწილი, რომელიც დაახლოებით გამოიყურება ასე: kDCXBwzSI",
|
||||
"Background Video": "ფონის ვიდეო",
|
||||
"YouTube Video ID": "YouTube-ის ვიდეოს ID",
|
||||
"Character Animation": "პერსონაჟის ანიმაცია",
|
||||
"Select the animation to play": "აირჩიეთ დასაკრავი ანიმაცია",
|
||||
"Animation": "ანიმაცია",
|
||||
"Character Model": "პერსონაჟის ანიმაცია",
|
||||
"character_desc": "აირჩიეთ, რომელი პერსონაჟით გსურთ თამაში. ამჟამად ერთადერთი პერსონაჟი, რომელიც Amica-ს გააჩნია, სავსეა ემოციებით. საჭიროების შემთხვევაში ჩატვირთეთ აქ თქვენი VRM-ები",
|
||||
"Load VRM": "VRM-ის ჩატვირთვა",
|
||||
"Chatbot Backend": "ჩატბოტის უკანაბოლო",
|
||||
"Chatbot_Backend_desc": "აირჩიეთ ჩეთბოტის სარეზერვო სისტემა გამოსაყენებლად. Echo უბრალოდ პასუხობს იმას, რასაც თქვენ აკრიფებთ, ის გამოიყენება ტესტირებისთვის და დემონსტრირებისთვის. ნაგულისხმევად Amica არის კონფიგურირებული, რომ გამოიყენოს ჩვენი დემო სერვერები, რომლებსაც შეიძლება დრო დასჭირდეს ჩატვირთვისას (უსაქმურობის დროს) ან გადატვირთული იყოს. თუ ის ნელა რეაგირებს, მიეცით დრო.",
|
||||
"Configure": "მორგება",
|
||||
"Click here to configure": "მოსარგებად დააწკაპუნეთ აქ",
|
||||
"Appearance": "გარეგნობა",
|
||||
"ChatBot": "ჩატბოტი",
|
||||
"Text-to-Speech": "ტექსტიდან-საუბარზე",
|
||||
"Speech-to-text": "საუბრიდან-ტექსტამდე",
|
||||
"Vision": "ხედვა",
|
||||
"Reset Settings": "პარამეტრების ჩამოყრა",
|
||||
"Community": "საზოგადოება",
|
||||
"Discord": "Discord",
|
||||
"Background Image": "ფონის სურათი",
|
||||
"ChatBot Backend": "ჩატბოტის უკანაბოლო",
|
||||
"Echo": "Echo",
|
||||
"ChatGPT": "ChatGPT",
|
||||
"LLama.cpp": "LLama.cpp",
|
||||
"Ollama": "Ollama",
|
||||
"KoboldAI": "KoboldAI",
|
||||
"Window.ai": "Window.ai",
|
||||
"Name": "სახელი",
|
||||
"System Prompt": "სისტემური მოთხოვნის ველი",
|
||||
"TTS": "TTS",
|
||||
"TTS Backend": "TTS-ის უკანაბოლო",
|
||||
"Piper": "Piper",
|
||||
"ElevenLabs": "ElevenLabs",
|
||||
"SpeechT5": "SpeechT5",
|
||||
"Coqui": "Coqui",
|
||||
"OpenAI": "OpenAI",
|
||||
"Vision Backend": "ხედვის უკანაბოლო",
|
||||
"STT Backend": "STT-ის უკანაბოლო",
|
||||
"Whisper (OpenAI)": "Whisper (OpenAI)",
|
||||
"Whisper.cpp": "Whisper.cpp",
|
||||
"Wake Word": "გაღვიძების სიტყვა",
|
||||
"Reset to Default": "ნაგულისხმევზე ჩამოყრა",
|
||||
"community_desc": "შემოუერთდით Amica-ის საზოგადოებას სხვებთან ერთად და გაეცანით მის ტექნოლოგიებს",
|
||||
"Telegram": "Telegram",
|
||||
"Twitter": "Twitter",
|
||||
"Documentation": "დოკუმენტაცია",
|
||||
"Settings": "მორგება",
|
||||
"piper_desc": "Piper-ის მორგება",
|
||||
"coqui_desc": "Conqui-ის მორგება",
|
||||
"not_using_alert": "ამჟამად თქვენ არ იყენებთ {{name}}-ს თქვენი {{what}}-ის უკანაბოლოდ. ეს პარამეტრები გამოყენებული არ იქნება",
|
||||
"API Key": "API-ის გასაღები",
|
||||
"Voice ID": "ხმის ID",
|
||||
"elevenLabs_desc": "ElevenLabs-ის მორგება",
|
||||
"kobold_cpp_desc": "KoboldCpp იოლი ტექსტის გენერაციის ხი-ია GGML და GGUF მოდელებისთვის",
|
||||
"API URL": "API-ის ბმული",
|
||||
"llama.cpp": "llama.cpp",
|
||||
"llama_cpp_desc": "LLama.cpp თავისუფალი და ღია კოდის მქონე ჩატბოტის უკანაბოლოა. სერვერი წესით კოდიდან თვითონ უნდა ააგოთ და საკუთარ კომპიუტერზე გაუშვათ. კოდის გადმოწერა შეგიძლიათ საიტიდან",
|
||||
"GitHub": "GitHub",
|
||||
"cfg_avatar_name_desc": "ავატარის სახელის დაყენება. ეს სახელია, რომელიც ჩატბოტის პასუხის გენერაციისას გამოიყენება",
|
||||
"ollama_desc": "Ollama საშუალებას გაძლევთ, LLM თქვენს კომპიუტერზე გაუშვათ. გადმოწერეთ ის საიტიდან",
|
||||
"ollama.ai": "ollama.ai",
|
||||
"Model": "მოდელი",
|
||||
"OpenAI TTS": "OpenAI TTS",
|
||||
"OpenAI_tts_desc": "OpenAI TTS-ის მორგება",
|
||||
"Voice": "ხმა",
|
||||
"Reset All Settings": "ყველა პარამეტრის ჩამოყრა",
|
||||
"SpeechT5_desc": "SpeechT5-ის მორგება",
|
||||
"STT": "STT",
|
||||
"STT_desc": "აირჩიეთ STT-ის უკანაბოლო",
|
||||
"System_Prompt_desc": "სისტემის შეყვანის ზოლის მორგება. ეს ის ზოლია, რომელიც ჩატბოტის პასუხის გენერაციისთვის გამოიყენება",
|
||||
"TTS_Backend_desc": "აირჩიეთ სასურველი TTS-ის უკანაბოლო. ნაგულისხმევად ეს ჩვენი TTS სერვერია. შეგიძლიათ, დააყენოთ ElevenLabs-ის ფასიანი სერვისი. უფასო, არაკომერციული საქმიანობისთვის, შეგიძლიათ SpeechT5 გამოიყენოთ. OpenAI TTS ღია კოდის მქონე TTS მოდელია. RVC ხმის ხელოვნური ეფექტების გამოყენების საშუალებას გაძლევთ. Piper უფასო TTS მოდელია. Coqui Local უფასო TTS მოდელია.",
|
||||
"Vision_Backend_desc": "აირჩიეთ Vision-ის უკანაბოლო",
|
||||
"Vision_System_Prompt_desc": "Vision-ის შეყვანის ზოლის მორგება. ეს ზოლია, რომელიც გამოსახულების აღწერის გენერაციისთვის გამოიყენება.",
|
||||
"Whisper_cpp_desc": "Whisper.cpp-ის მორგება",
|
||||
"URL": "URL",
|
||||
"Whisper_OpenAI_desc": "Whisper-ის (OpenAI) მორგება",
|
||||
"OpenAI URL": "OpenAI-ის ბმული",
|
||||
"Error!": "შეცდომა!",
|
||||
"Something went wrong.": "რაღაც არასწორად წავიდა.",
|
||||
"Try again later.": "მოგვიანებით სცადეთ.",
|
||||
"Click here": "აქ დააწკაპუნეთ",
|
||||
"to return to homepage.": "საწყის გვერდზე დასაბრუნებლად.",
|
||||
"Import": "შემოტანა",
|
||||
"Description": "აღწერა",
|
||||
"Vision System Prompt": "Vision-ის სისტემური შეყვანის ველი",
|
||||
"Background Url": "ფონის URL",
|
||||
"Youtube Video Id": "Youtube-ის ვიდეოს Id",
|
||||
"VRM Url": "VRM Url",
|
||||
"Animation Url": "ანიმაციის Url",
|
||||
"Voice Url": "ხმის Url",
|
||||
"Upload VRM": "VRM-ის ატვირთვა",
|
||||
"VRM upload message": "VRM შეტყობინების ატვირთვა",
|
||||
"VRM local share message": "VRM ლოკალური გაზიარების შეტყობინება",
|
||||
"Import character": "სიმბოლოს შემოტნა",
|
||||
"Wrong place?": "ადგილი არასწორია?",
|
||||
"Go home": "საწყის გვერდზე გადასვლა",
|
||||
"Please enter the code you received from the share page.": "შეიყვანეთ კოდი, რომელიც გაზიარების გვერდზე მიიღეთ.",
|
||||
"Code": "კოდი",
|
||||
"Provide a description of the character": "შეიყვანეთ სიმბოლოსა ღწერა",
|
||||
"Background URL": "ფონის URL",
|
||||
"Example": "მაგალითი",
|
||||
"Save Character": "სიმბოლოს შენახვა",
|
||||
"Share this code (click to copy):": "ამ კოდის გაზიარება (დააწკაპუნეთ დასაკოპირებლად):",
|
||||
"Or, you can share this direct link:": "ან, შეგიძლიათ, ეს პირდაპირი ბმული გააზიაროთ:",
|
||||
"Conversation": "საუბარი",
|
||||
"Debug": "გამართვა",
|
||||
"Import Character": "სიმბოლოს შემოტნა",
|
||||
"Copy URL": "URL-ის კოპირება",
|
||||
"Preview": "მინიატურა",
|
||||
"None": "არცერთი",
|
||||
"Speech-to-Text": "საუბარი-ტექსტამდე",
|
||||
"Set time before bot go idle": "დააყენეთ დრო, როდის გადავა ბოტი უქმე რეჟიმში",
|
||||
"time before bot go idle": "დრო, სანამ ბოტი უქმე რეჟიმში გადავა",
|
||||
"Wake word": "გაღვიძების სიტყვა",
|
||||
"Specify": "მიუთითეთ",
|
||||
"sec": "წმ",
|
||||
"Use": "გამოყენება",
|
||||
"Set": "დაყენება",
|
||||
"Enabled": "ჩართულია",
|
||||
"STOP SEQUENCE": "STOP SEQUENCE"
|
||||
}
|
||||
@ -1,37 +1,10 @@
|
||||
import '@/i18n';
|
||||
|
||||
import '@rainbow-me/rainbowkit/styles.css';
|
||||
import "@/styles/globals.css";
|
||||
import "@charcoal-ui/icons";
|
||||
import type { AppProps } from "next/app";
|
||||
import {
|
||||
getDefaultConfig,
|
||||
RainbowKitProvider,
|
||||
} from '@rainbow-me/rainbowkit';
|
||||
import { WagmiProvider } from 'wagmi';
|
||||
import { arbitrum } from 'wagmi/chains';
|
||||
import {
|
||||
QueryClientProvider,
|
||||
QueryClient,
|
||||
} from "@tanstack/react-query";
|
||||
|
||||
const config = getDefaultConfig({
|
||||
appName: 'arbius.heyamica.com',
|
||||
projectId: process.env.NEXT_PUBLIC_WALLETCONNECT_PROJECT_ID!,
|
||||
chains: [arbitrum],
|
||||
ssr: true, // If your dApp uses server side rendering (SSR)
|
||||
});
|
||||
|
||||
const queryClient = new QueryClient();
|
||||
|
||||
export default function App({ Component, pageProps }: AppProps) {
|
||||
return (
|
||||
<WagmiProvider config={config}>
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<RainbowKitProvider>
|
||||
<Component {...pageProps} />
|
||||
</RainbowKitProvider>
|
||||
</QueryClientProvider>
|
||||
</WagmiProvider>
|
||||
<Component {...pageProps} />
|
||||
);
|
||||
}
|
||||
|
||||
@ -46,7 +46,6 @@ import { LoadingProgress } from "@/components/loadingProgress";
|
||||
import { DebugPane } from "@/components/debugPane";
|
||||
import { Settings } from "@/components/settings";
|
||||
import { EmbeddedWebcam } from "@/components/embeddedWebcam";
|
||||
// import { Moshi } from "@/features/moshi/components/Moshi";
|
||||
import { Moshi } from "@/features/moshi/components/Moshi";
|
||||
|
||||
import { ViewerContext } from "@/features/vrmViewer/viewerContext";
|
||||
@ -64,6 +63,7 @@ import { ChatModeText } from "@/components/chatModeText";
|
||||
import { TimestampedPrompt } from "@/features/amicaLife/eventHandler";
|
||||
import { handleChatLogs } from "@/features/externalAPI/externalAPI";
|
||||
import { VerticalSwitchBox } from "@/components/switchBox";
|
||||
import { ThoughtText } from "@/components/thoughtText";
|
||||
|
||||
const m_plus_2 = M_PLUS_2({
|
||||
variable: "--font-m-plus-2",
|
||||
@ -126,6 +126,7 @@ export default function Home() {
|
||||
const [chatLog, setChatLog] = useState<Message[]>([]);
|
||||
const [assistantMessage, setAssistantMessage] = useState("");
|
||||
const [userMessage, setUserMessage] = useState("");
|
||||
const [thoughtMessage, setThoughtMessage] = useState("");
|
||||
const [shownMessage, setShownMessage] = useState<Role>("system");
|
||||
const [subconciousLogs, setSubconciousLogs] = useState<TimestampedPrompt[]>([]);
|
||||
|
||||
@ -171,20 +172,20 @@ export default function Home() {
|
||||
} else {
|
||||
document.body.style.backgroundImage = `url(${config("bg_url")})`;
|
||||
}
|
||||
// Temp Disable : WebXR
|
||||
// if (window.navigator.xr && window.navigator.xr.isSessionSupported) {
|
||||
// let deviceInfo = detectVRHeadset();
|
||||
// setIsVRHeadset(deviceInfo.isVRDevice);
|
||||
|
||||
if (window.navigator.xr && window.navigator.xr.isSessionSupported) {
|
||||
let deviceInfo = detectVRHeadset();
|
||||
setIsVRHeadset(deviceInfo.isVRDevice);
|
||||
|
||||
window.navigator.xr.isSessionSupported('immersive-ar').then((supported) => {
|
||||
console.log('ar supported', supported);
|
||||
setIsARSupported(supported);
|
||||
});
|
||||
window.navigator.xr.isSessionSupported('immersive-vr').then((supported) => {
|
||||
console.log('vr supported', supported);
|
||||
setIsVRSupported(supported);
|
||||
});
|
||||
}
|
||||
// window.navigator.xr.isSessionSupported('immersive-ar').then((supported) => {
|
||||
// console.log('ar supported', supported);
|
||||
// setIsARSupported(supported);
|
||||
// });
|
||||
// window.navigator.xr.isSessionSupported('immersive-vr').then((supported) => {
|
||||
// console.log('vr supported', supported);
|
||||
// setIsVRSupported(supported);
|
||||
// });
|
||||
// }
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
@ -303,6 +304,7 @@ export default function Home() {
|
||||
setChatLog,
|
||||
setUserMessage,
|
||||
setAssistantMessage,
|
||||
setThoughtMessage,
|
||||
setShownMessage,
|
||||
setChatProcessing,
|
||||
setChatSpeaking,
|
||||
@ -339,7 +341,7 @@ export default function Home() {
|
||||
)}>
|
||||
{showStreamWindow &&
|
||||
|
||||
<div className="fixed top-4 right-4 w-[200px] h-[150px] z-20">
|
||||
<div className="fixed top-1/3 right-4 w-[200px] h-[150px] z-0">
|
||||
<video
|
||||
ref={videoRef}
|
||||
autoPlay
|
||||
@ -376,7 +378,7 @@ export default function Home() {
|
||||
)}
|
||||
</VrmStoreProvider>
|
||||
|
||||
{ config("chatbot_backend") !== "moshi" && <MessageInputContainer isChatProcessing={chatProcessing} />}
|
||||
<MessageInputContainer isChatProcessing={chatProcessing} />
|
||||
|
||||
{/* main menu */}
|
||||
<div className="absolute z-10 m-2">
|
||||
@ -467,7 +469,8 @@ export default function Home() {
|
||||
/>
|
||||
)}
|
||||
|
||||
<MenuButton
|
||||
{/* Temp Disable : WebXR */}
|
||||
{/*<MenuButton
|
||||
large={isVRHeadset}
|
||||
icon={CubeTransparentIcon}
|
||||
disabled={!isARSupported}
|
||||
@ -481,7 +484,7 @@ export default function Home() {
|
||||
disabled={!isVRSupported}
|
||||
onClick={() => toggleXR('immersive-vr')}
|
||||
label="Virtual Reality"
|
||||
/>
|
||||
/>*/}
|
||||
|
||||
<MenuButton
|
||||
large={isVRHeadset}
|
||||
@ -490,7 +493,8 @@ export default function Home() {
|
||||
label="debug"
|
||||
/>
|
||||
|
||||
{ showChatMode ? (
|
||||
{/* Temp Disable : WebXR */}
|
||||
{/* { showChatMode ? (
|
||||
<MenuButton
|
||||
large={isVRHeadset}
|
||||
icon={Squares2X2Icon}
|
||||
@ -506,7 +510,7 @@ export default function Home() {
|
||||
onClick={toggleChatMode}
|
||||
label="show chat mode"
|
||||
/>
|
||||
)}
|
||||
)} */}
|
||||
|
||||
<div className="flex flex-row items-center space-x-2">
|
||||
<VerticalSwitchBox
|
||||
@ -536,17 +540,6 @@ export default function Home() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/*
|
||||
<Moshi
|
||||
workerAddr="https://orcsza38j78yrh-8998.proxy.runpod.net/"
|
||||
workerAuthId="amica"
|
||||
audioContext={null}
|
||||
worklet={null}
|
||||
onConversationEnd={() => {}}
|
||||
isBypass={false}
|
||||
/>
|
||||
*/}
|
||||
|
||||
{showChatLog && <ChatLog messages={chatLog} />}
|
||||
|
||||
{/* Normal chat text */}
|
||||
@ -561,6 +554,9 @@ export default function Home() {
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Thought text */}
|
||||
{thoughtMessage !== "" && <ThoughtText message={thoughtMessage}/>}
|
||||
|
||||
{/* Chat mode text */}
|
||||
{showChatMode && <ChatModeText messages={chatLog}/>}
|
||||
|
||||
|
||||
@ -88,6 +88,7 @@ export async function askLLM(
|
||||
let sentences = new Array<string>();
|
||||
let aiTextLog = "";
|
||||
let tag = "";
|
||||
let isThinking = false;
|
||||
let rolePlay = "";
|
||||
let result = "";
|
||||
|
||||
@ -119,6 +120,7 @@ export async function askLLM(
|
||||
aiTextLog,
|
||||
receivedMessage,
|
||||
tag,
|
||||
isThinking,
|
||||
rolePlay,
|
||||
callback: (aiTalks: Screenplay[]): boolean => {
|
||||
// Generate & play audio for each sentence, display responses
|
||||
@ -126,10 +128,15 @@ export async function askLLM(
|
||||
console.log('wrong stream idx');
|
||||
return true; // should break
|
||||
}
|
||||
chat.ttsJobs.enqueue({
|
||||
screenplay: aiTalks[0],
|
||||
streamIdx: currentStreamIdx,
|
||||
});
|
||||
if (!isThinking) {
|
||||
chat.ttsJobs.enqueue({
|
||||
screenplay: aiTalks[0],
|
||||
streamIdx: currentStreamIdx,
|
||||
});
|
||||
}
|
||||
|
||||
// thought bubble
|
||||
chat.thoughtBubbleMessage(isThinking, aiTalks[0].text);
|
||||
|
||||
if (! firstSentenceEncountered) {
|
||||
console.timeEnd('performance_time_to_first_sentence');
|
||||
@ -144,6 +151,7 @@ export async function askLLM(
|
||||
aiTextLog = proc.aiTextLog;
|
||||
receivedMessage = proc.receivedMessage;
|
||||
tag = proc.tag;
|
||||
isThinking = proc.isThinking;
|
||||
rolePlay = proc.rolePlay;
|
||||
if (proc.shouldBreak) {
|
||||
break;
|
||||
|
||||
@ -20,7 +20,6 @@ export const defaults = {
|
||||
show_introduction: process.env.NEXT_PUBLIC_SHOW_INTRODUCTION ?? 'true',
|
||||
show_arbius_introduction: process.env.NEXT_PUBLIC_SHOW_ARBIUS_INTRODUCTION ?? 'false',
|
||||
show_add_to_homescreen: process.env.NEXT_PUBLIC_SHOW_ADD_TO_HOMESCREEN ?? 'true',
|
||||
scenario_url: process.env.NEXT_PUBLIC_SCENARIO_URL ?? '/scenarios/test1.js',
|
||||
bg_color: process.env.NEXT_PUBLIC_BG_COLOR ?? '',
|
||||
bg_url: process.env.NEXT_PUBLIC_BG_URL ?? '/bg/bg-room2.jpg',
|
||||
vrm_url: process.env.NEXT_PUBLIC_VRM_HASH ?? '/vrm/AvatarSample_A.vrm',
|
||||
@ -78,6 +77,8 @@ export const defaults = {
|
||||
rvc_protect: process.env.NEXT_PUBLIC_RVC_PROTECT ?? '0.33',
|
||||
coquiLocal_url: process.env.NEXT_PUBLIC_COQUILOCAL_URL ?? 'http://localhost:5002',
|
||||
coquiLocal_voiceid: process.env.NEXT_PUBLIC_COQUILOCAL_VOICEID ?? 'p240',
|
||||
kokoro_url: process.env.NEXT_PUBLIC_KOKORO_URL ?? 'http://localhost:8080',
|
||||
kokoro_voice: process.env.NEXT_PUBLIC_KOKORO_VOICE ?? 'af_bella',
|
||||
piper_url: process.env.NEXT_PUBLIC_PIPER_URL ?? 'https://i-love-amica.com:5000/tts',
|
||||
elevenlabs_apikey: process.env.NEXT_PUBLIC_ELEVENLABS_APIKEY ??'',
|
||||
elevenlabs_voiceid: process.env.NEXT_PUBLIC_ELEVENLABS_VOICEID ?? '21m00Tcm4TlvDq8ikWAM',
|
||||
|
||||
@ -5,6 +5,7 @@ export type ProcessResponseRetVal = {
|
||||
aiTextLog: string;
|
||||
receivedMessage: string;
|
||||
tag: string;
|
||||
isThinking: boolean;
|
||||
rolePlay: string;
|
||||
shouldBreak: boolean;
|
||||
}
|
||||
@ -18,6 +19,7 @@ export function processResponse({
|
||||
aiTextLog,
|
||||
receivedMessage,
|
||||
tag,
|
||||
isThinking,
|
||||
rolePlay,
|
||||
callback,
|
||||
}: {
|
||||
@ -25,11 +27,22 @@ export function processResponse({
|
||||
aiTextLog: string,
|
||||
receivedMessage: string,
|
||||
tag: string,
|
||||
isThinking: boolean,
|
||||
rolePlay: string,
|
||||
callback: (aiTalks: Screenplay[]) => boolean,
|
||||
}): ProcessResponseRetVal {
|
||||
let shouldBreak = false;
|
||||
|
||||
const thinkTagMatch = receivedMessage.match(/<\/?think>/);
|
||||
if (thinkTagMatch && thinkTagMatch[0]) {
|
||||
if (thinkTagMatch[0] === "</think>") {
|
||||
isThinking = false;
|
||||
} else {
|
||||
isThinking = true;
|
||||
}
|
||||
receivedMessage = receivedMessage.slice(thinkTagMatch[0].length);
|
||||
}
|
||||
|
||||
// Detection of tag part of reply content
|
||||
const tagMatch = receivedMessage.match(/^\[(.*?)\]/);
|
||||
if (tagMatch && tagMatch[0]) {
|
||||
@ -68,6 +81,7 @@ export function processResponse({
|
||||
aiTextLog,
|
||||
receivedMessage,
|
||||
tag,
|
||||
isThinking,
|
||||
rolePlay,
|
||||
shouldBreak,
|
||||
}
|
||||
@ -85,6 +99,7 @@ export function processResponse({
|
||||
aiTextLog,
|
||||
receivedMessage,
|
||||
tag,
|
||||
isThinking,
|
||||
rolePlay,
|
||||
shouldBreak,
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user