Compare commits

..

2 Commits

Author SHA1 Message Date
Joey Wong
b368be8e6b
Re-enabled WebXR
Re-WebXR
2025-02-26 16:39:35 +07:00
flukexp
c38717277d renewable webxr 2025-02-25 12:38:05 +07:00
36 changed files with 10526 additions and 1272 deletions

View File

@ -3,6 +3,7 @@ NEXT_PUBLIC_AMICA_API_URL=https://store.heyamica.com
NEXT_PUBLIC_AMICA_STORAGE_URL=https://vrm.heyamica.com/file/amica-vrm
NEXT_PUBLIC_SUPABASE_URL=https://kzfcwefqffysqsjrlyld.supabase.co
NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Imt6ZmN3ZWZxZmZ5c3FzanJseWxkIiwicm9sZSI6ImFub24iLCJpYXQiOjE3MDEyNzA5NjUsImV4cCI6MjAxNjg0Njk2NX0.z9zm2mbJu6RkdZ6zkCUltThIF43-ava_bLvk9qFQsiA
NEXT_PUBLIC_WALLETCONNECT_PROJECT_ID=9f8cee238fde4f0011880ecef0616c88
X_API_KEY="X_API_KEY"
X_API_SECRET="X_API_SECRET"

View File

@ -35,7 +35,6 @@
* [Using Coqui Local](./guides/using-coqui.md)
* [Using Piper ](./guides/using-piper.md)
* [Using Alltalk TTS](./guides/using-alltalk.md)
* [Using Kokoro TTS](./guides/using-kokoro.md)
* [Using RVC](./guides/using-rvc.md)

View File

@ -1,51 +0,0 @@
---
title: Using Kokoro
order: 16
---
Navigate to [Kokoro TTS GitHub repository](https://github.com/hexgrad/kokoro).
## Setting Up Kokoro TTS Server
### Clone the Repository
```bash
git clone https://github.com/flukexp/kokoro-tts.git
cd kokoro-tts
```
### Create a Virtual Environment
```bash
python -m venv venv
source venv/bin/activate # On Windows, use `venv\Scripts\activate`
```
### Install Dependencies
```bash
pip install -r requirements.txt
```
## Running the Server
### Start the FastAPI Server
```bash
python server.py
```
## Make sure Kokoro is enabled for TTS:
```bash
Settings -> Text-to-Speech -> TTS Backend -> Kokoro
```
## Set the voice
```bash
Settings -> Text-to-Speech -> Kokoro -> Voice
```
## Using Kokoro with OpenAI TTS
You can use Kokoro by choosing OpenAI and configuring your Kokoro endpoint and voice.
### Notes
- Kokoro TTS can be used as a local text-to-speech backend in your application.
- If you want to explore more models or functionalities, refer to the official [Kokoro TTS GitHub repository](https://github.com/hexgrad/kokoro).

View File

@ -8,7 +8,7 @@ To test Moshi, you need to set up and run the Moshi Server on Runpod (Or your ru
---
### **Step 1: Set Up Moshi Server**
### **Step 1: Set Up and Running Moshi Server for Amica**
1. Login to the Terminal on your instance, whether it is your own server or a Runpod instance (If you don't have a good GPU)
1. **Clone the Moshi Server**
```bash
@ -34,14 +34,9 @@ To test Moshi, you need to set up and run the Moshi Server on Runpod (Or your ru
.\venv\Scripts\Activate
```
4. **Install dependencies**
```bash
pip install -r requirements.txt
```
---
### **Step 2: Start the Moshi Server**
### **Step 2: Run the Moshi Server**
Start the Moshi server (For Runpod):
```bash
@ -69,6 +64,6 @@ Then go to Settings > Chatbot Backend > Moshi, and then insert the correct URL f
- Ensure that **Python and pip** are installed before proceeding. **Edit the model URLs in the main python script if you want to use a different model from Kyutai.**
---
Replace the `{port-number}` in the URL with the actual port number.
Replace the `{port-number}` in the URL with the actual port number (e.g 3000 for Amica and 8000 for Moshi).
---

View File

@ -2,7 +2,7 @@
// run i18next
export default {
defaultNamespace: 'common',
locales: ['en', 'zh', 'de', 'ka'],
locales: ['en', 'zh', 'de'],
output: 'src/i18n/locales/$LOCALE/$NAMESPACE.json',
input: ["src/**/*.{ts,tsx}"],
keepRemoved: true,

10422
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -28,11 +28,13 @@
"@mkkellogg/gaussian-splats-3d": "^0.4.5",
"@next/third-parties": "^14.2.16",
"@pixiv/three-vrm": "^3.1.2",
"@rainbow-me/rainbowkit": "^2.2.0",
"@ricky0123/vad-react": "^0.0.24",
"@sentry/nextjs": "^8.34.0",
"@supabase/supabase-js": "^2.45.4",
"@tabler/icons-react": "^3.19.0",
"@tailwindcss/forms": "^0.5.9",
"@tanstack/react-query": "^5.59.16",
"@testing-library/jest-dom": "^6.5.0",
"@webxr-input-profiles/assets": "^1.0.16",
"@webxr-input-profiles/motion-controllers": "^1.0.0",
@ -64,6 +66,8 @@
"three.quarks": "^0.15.5",
"twitter-api-v2": "^1.19.0",
"typescript-collections": "^1.3.3",
"viem": "^2.21.37",
"wagmi": "^2.12.25",
"wavefile": "^11.0.0",
"webm-duration-fix": "^1.0.4",
"window.ai": "^0.2.4",

View File

@ -17,7 +17,7 @@ class Scenario {
await $.loadVrm(
// 'https://vrm.heyamica.com/file/amica-vrm/82754e287e0b26b5d7a1fd223ed0fd5debcabed81f36549fb2c16b201f3e5ca9',
'/vrm/AvatarSample_B.vrm',
'/vrm/AvatarSample_A.vrm',
(progress) => {
console.log(`loading model ${progress}`);
},
@ -49,6 +49,6 @@ class Scenario {
}
update(delta) {
console.log('update', this.x);
// console.log('update', this.x);
}
}

View File

@ -1,3 +1,6 @@
import { useAccount } from 'wagmi'
import { ConnectButton } from '@rainbow-me/rainbowkit';
import { useState } from "react";
import { useTranslation } from 'react-i18next';
export const ArbiusIntroduction = ({ open, close }: {
@ -5,8 +8,10 @@ export const ArbiusIntroduction = ({ open, close }: {
close: () => void;
}) => {
const { t } = useTranslation();
const { address } = useAccount();
if (!open) {
if (! open) {
return null;
}
@ -24,15 +29,23 @@ export const ArbiusIntroduction = ({ open, close }: {
</p>
</div>
<div className="my-8 flex space-x-4">
<ConnectButton />
</div>
<div className="my-8">
<button
onClick={() => {
close();
}}
className="ml-3 inline-flex items-center rounded-md bg-white px-2.5 py-1.5 text-sm font-semibold text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 hover:bg-gray-50"
>
{t("Continue to Amica")}
</button>
{address && (
<button
onClick={() => {
close();
}}
className="ml-3 inline-flex items-center rounded-md bg-white px-2.5 py-1.5 text-sm font-semibold text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 hover:bg-gray-50"
>
{t("Continue to Amica")}
</button>
)}
{(!address) && (
<p>{t("Please connect your wallet to continue.")}</p>
)}
</div>
</div>
</div>

View File

@ -14,7 +14,6 @@ import { whispercpp } from "@/features/whispercpp/whispercpp";
import { config } from "@/utils/config";
import { WaveFile } from "wavefile";
import { AmicaLifeContext } from "@/features/amicaLife/amicaLifeContext";
import { AudioControlsContext } from "@/features/moshi/components/audioControlsContext";
export default function MessageInput({
@ -37,8 +36,6 @@ export default function MessageInput({
const { chat: bot } = useContext(ChatContext);
const { alert } = useContext(AlertContext);
const { amicaLife } = useContext(AmicaLifeContext);
const { audioControls: moshi } = useContext(AudioControlsContext);
const [ moshiMuted, setMoshiMuted] = useState(moshi.isMuted());
const vad = useMicVAD({
startOnLoad: false,
@ -215,33 +212,20 @@ export default function MessageInput({
<div className="grid grid-flow-col grid-cols-[min-content_1fr_min-content] gap-[8px]">
<div>
<div className='flex flex-col justify-center items-center'>
{config("chatbot_backend") === "moshi" ? (
<IconButton
iconName={!moshiMuted ? "24/PauseAlt" : "24/Microphone"}
className="bg-secondary hover:bg-secondary-hover active:bg-secondary-press disabled:bg-secondary-disabled"
isProcessing={moshiMuted && moshi.getRecorder() != null}
disabled={!moshi.getRecorder()}
onClick={() => {
moshi.toggleMute();
setMoshiMuted(!moshiMuted);
}}
/>
) : (
<IconButton
<IconButton
iconName={vad.listening ? "24/PauseAlt" : "24/Microphone"}
className="bg-secondary hover:bg-secondary-hover active:bg-secondary-press disabled:bg-secondary-disabled"
isProcessing={vad.userSpeaking}
disabled={config('stt_backend') === 'none' || vad.loading || Boolean(vad.errored)}
onClick={vad.toggle}
/>
)}
</div>
</div>
<input
type="text"
ref={inputRef}
placeholder={config("chatbot_backend") === "moshi" ? "Disabled in moshi chatbot" : "Write message here..."}
placeholder="Write message here..."
onChange={handleInputChange}
onKeyDown={(e) => {
if (e.key === "Enter") {
@ -256,7 +240,7 @@ export default function MessageInput({
clickedSendButton();
}
}}
disabled={config("chatbot_backend") === "moshi"}
disabled={false}
className="disabled block w-full rounded-md border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-1 focus:ring-inset focus:ring-gray-400 sm:text-sm sm:leading-6"
value={userMessage}
@ -268,7 +252,7 @@ export default function MessageInput({
iconName="24/Send"
className="ml-2 bg-secondary hover:bg-secondary-hover active:bg-secondary-press disabled:bg-secondary-disabled"
isProcessing={isChatProcessing || transcriber.isBusy}
disabled={isChatProcessing || !userMessage || transcriber.isModelLoading || config("chatbot_backend") === "moshi"}
disabled={isChatProcessing || !userMessage || transcriber.isModelLoading}
onClick={clickedSendButton}
/>
</div>

View File

@ -72,7 +72,6 @@ import { AmicaLifePage } from "./settings/AmicaLifePage";
import { useVrmStoreContext } from "@/features/vrmStore/vrmStoreContext";
import { OpenRouterSettings } from "./settings/OpenRouterSettingsPage";
import { ExternalAPIPage } from "./settings/ExternalAPIPage";
import { KokoroSettingsPage } from "./settings/KokoroSettingsPage";
export const Settings = ({
@ -136,9 +135,6 @@ export const Settings = ({
const [localXTTSUrl, setLocalXTTSUrl] = useState(config("localXTTS_url"));
const [kokoroUrl, setKokoroUrl] = useState(config("kokoro_url"));
const [kokoroVoice, setKokoroVoice] = useState(config("kokoro_voice"));
const [visionBackend, setVisionBackend] = useState(config("vision_backend"));
const [visionLlamaCppUrl, setVisionLlamaCppUrl] = useState(config("vision_llamacpp_url"));
const [visionOllamaUrl, setVisionOllamaUrl] = useState(config("vision_ollama_url"));
@ -256,7 +252,7 @@ export const Settings = ({
useEffect(() => {
// Change the chatbot to 'llamacpp' if Amica Life is enabled and previous chatbot was 'echo'
if (amicaLifeEnabled && ["echo", "moshi"].includes(config("chatbot_backend"))) {
if (amicaLifeEnabled && chatbotBackend === "echo") {
setAmicaLifeEnabled(false);
}
}, [chatbotBackend, amicaLifeEnabled]);
@ -288,7 +284,6 @@ export const Settings = ({
rvcUrl,rvcEnabled,rvcModelName,rvcIndexPath,rvcF0upKey,rvcF0Method,rvcIndexRate,rvcFilterRadius,,rvcResampleSr,rvcRmsMixRate,rvcProtect,
coquiLocalUrl,coquiLocalVoiceId,
localXTTSUrl,
kokoroUrl, kokoroVoice,
visionBackend,
visionLlamaCppUrl,
visionOllamaUrl, visionOllamaModel,
@ -373,7 +368,7 @@ export const Settings = ({
case 'tts':
return <MenuPage
keys={["tts_backend", "elevenlabs_settings", "speecht5_settings", "coquiLocal_settings", "openai_tts_settings", "piper_settings", "localXTTS_settings", "kokoro_settings", "rvc_settings"]}
keys={["tts_backend", "elevenlabs_settings", "speecht5_settings", "coquiLocal_settings", "openai_tts_settings", "piper_settings", "localXTTS_settings", "rvc_settings"]}
menuClick={handleMenuClick} />;
case 'stt':
@ -588,15 +583,6 @@ export const Settings = ({
setSettingsUpdated={setSettingsUpdated}
/>
case 'kokoro_settings':
return <KokoroSettingsPage
kokoroUrl={kokoroUrl}
kokoroVoice={kokoroVoice}
setKokoroVoice={setKokoroVoice}
setKokoroUrl={setKokoroUrl}
setSettingsUpdated={setSettingsUpdated}
/>
case 'rvc_settings':
return <RVCSettingsPage
rvcUrl={rvcUrl}

View File

@ -99,7 +99,7 @@ export function AmicaLifePage({
<SwitchBox
value={amicaLifeEnabled}
label={`${t("Amica Life")} ${t("Enabled")} ${t("(Disable to improve performance)")}`}
disabled={["echo", "moshi"].includes(config("chatbot_backend")) }
disabled={config("chatbot_backend") === "echo"}
onChange={(value: boolean) => {
setAmicaLifeEnabled(value);
updateConfig("amica_life_enabled", value.toString());

View File

@ -61,7 +61,6 @@ export function CoquiLocalSettingsPage({
<li className="py-4">
<FormRow label={t("Voice ID")}>
<select
className="mt-2 block w-full rounded-md border-0 py-1.5 pl-3 pr-10 text-gray-900 ring-1 ring-inset ring-gray-300 focus:ring-2 focus:ring-indigo-600 sm:text-sm sm:leading-6"
value={coquiLocalVoiceId}
onChange={(event: React.ChangeEvent<any>) => {
event.preventDefault();

View File

@ -1,86 +0,0 @@
import React, { useState, useEffect } from 'react';
import { useTranslation } from 'react-i18next';
import { BasicPage, FormRow, NotUsingAlert } from './common';
import { TextInput } from "@/components/textInput";
import { config, updateConfig } from "@/utils/config";
import { kokoroVoiceList } from '@/features/kokoro/kokoro';
export function KokoroSettingsPage({
kokoroUrl,
kokoroVoice,
setKokoroUrl,
setKokoroVoice,
setSettingsUpdated,
}: {
kokoroUrl: string;
kokoroVoice: string;
setKokoroUrl: (key: string) => void;
setSettingsUpdated: (updated: boolean) => void;
setKokoroVoice: (key: string) => void;
}) {
const { t } = useTranslation();
const [voiceList, setVoiceList] = useState<{ key: string; label: string }[]>([]);
useEffect(() => {
async function fetchVoiceList() {
try {
const data = await kokoroVoiceList();
if (data && data.voices) {
const formattedVoices = data.voices.map((voice: string) => ({
key: voice,
label: voice,
}));
setVoiceList(formattedVoices);
}
} catch (error) {
console.error("Error fetching kokoro voice list:", error);
}
}
fetchVoiceList();
}, []);
return (
<BasicPage
title={t("Kokoro") + " "+ t("Settings")}
description={t("kokoro_desc", "Configure Kokoro")}
>
{ config("tts_backend") !== "kokoro" && (
<NotUsingAlert>
{t("not_using_alert", "You are not currently using {{name}} as your {{what}} backend. These settings will not be used.", {name: t("Kokoro"), what: t("TTS")})}
</NotUsingAlert>
) }
<ul role="list" className="divide-y divide-gray-100 max-w-xs">
<li className="py-4">
<FormRow label={t("URL")}>
<TextInput
value={kokoroUrl}
onChange={(event: React.ChangeEvent<any>) => {
setKokoroUrl(event.target.value);
updateConfig("kokoro_url", event.target.value);
setSettingsUpdated(true);
}}
/>
</FormRow>
</li>
<li className="py-4">
<FormRow label={t("Voices")}>
<select
className="mt-2 block w-full rounded-md border-0 py-1.5 pl-3 pr-10 text-gray-900 ring-1 ring-inset ring-gray-300 focus:ring-2 focus:ring-indigo-600 sm:text-sm sm:leading-6"
value={kokoroVoice}
onChange={(event: React.ChangeEvent<any>) => {
event.preventDefault();
setKokoroVoice(event.target.value);
updateConfig("kokoro_voice", event.target.value);
setSettingsUpdated(true);
}}
>
{voiceList.map((voice) => (
<option key={voice.key} value={voice.key}>{t(voice.label)}</option>
))}
</select>
</FormRow>
</li>
</ul>
</BasicPage>
);
}

View File

@ -11,7 +11,6 @@ const ttsEngines = [
{key: "localXTTS", label: "Alltalk TTS"}, // Our local TTS endpoint (XTTS based)
{key: "piper", label: "Piper"},
{key: "coquiLocal", label: "Coqui Local"},
{key: "kokoro", label: "Kokoro"},
];
function idToTitle(id: string): string {
@ -58,7 +57,7 @@ export function TTSBackendPage({
</select>
</FormRow>
</li>
{ ["elevenlabs", "speecht5", "openai_tts", "piper", "coquiLocal", "localXTTS", "kokoro"].includes(ttsBackend) && (
{ ["elevenlabs", "speecht5", "openai_tts", "piper", "coquiLocal", "localXTTS"].includes(ttsBackend) && (
<li className="py-4">
<FormRow label={`${t("Configure")} ${t(idToTitle(ttsBackend))}`}>
<button

View File

@ -171,16 +171,15 @@ export function getIconFromPage(page: string): JSX.Element {
case 'elevenlabs_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'speecht5_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'openai_tts_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'piper_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'rvc_settings': return <CogIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'piper_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'rvc_settings': return <CogIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'coquiLocal_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'localXTTS_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'kokoro_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'localXTTS_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'stt_backend': return <PencilSquareIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'stt_wake_word': return <MoonIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'stt_backend': return <PencilSquareIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'stt_wake_word': return <MoonIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'whisper_openai_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'whispercpp_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'whispercpp_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'vision_backend': return <EyeDropperIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
case 'vision_llamacpp_settings': return <AdjustmentsHorizontalIcon className="h-5 w-5 flex-none text-gray-800" aria-hidden="true" />;
@ -232,7 +231,6 @@ function getLabelFromPage(page: string): string {
case 'rvc_settings': return t('RVC');
case 'coquiLocal_settings': return t('Coqui Local');
case 'localXTTS_settings': return t('Alltalk');
case 'kokoro_settings': return t('Kokoro');
case 'vision_backend': return t('Vision Backend');
case 'vision_llamacpp_settings': return t('LLama.cpp');

View File

@ -110,7 +110,7 @@ export const VerticalSwitchBox = ({
};
return (
<Switch.Group as="div" className="flex flex-col items-center">
<Switch.Group as="div" className="flex flex-col items-center space-y-3">
<Switch
checked={enabled}
onChange={handleChange}

View File

@ -1,57 +0,0 @@
import React, { useEffect, useRef, useState } from "react";
import { clsx } from "clsx";
import { config } from "@/utils/config";
import { IconButton } from "./iconButton";
export const ThoughtText = ({ message }: { message: string }) => {
const scrollRef = useRef<HTMLDivElement>(null);
const [expanded, setExpanded] = useState(false);
// Remove emotion tags
const cleanMessage = message.replace(/\[(.*?)\]/g, "");
useEffect(() => {
scrollRef.current?.scrollIntoView({
behavior: "smooth",
block: "center",
});
});
return (
<div
className={clsx(
"absolute top-4 right-3 z-40 transition-all duration-500 ease-in-out opacity-70 -translate-y-4"
)}
style={{
filter: "drop-shadow(0 2px 4px rgba(0, 0, 0, 0.1))",
maxWidth: "280px",
minWidth: "220px"
}}
>
<div className="bg-white/80 backdrop-blur-sm rounded-lg overflow-hidden border border-blue-100 shadow-md">
<div className="bg-blue-500/90 text-white px-3 py-1.5 flex items-center justify-between text-xs">
<span className="font-medium tracking-wide">
{`${config('name').toUpperCase()} THINKS`}
</span>
<IconButton
iconName="24/FrameSize"
className="bg-transparent hover:bg-blue-600/80 active:bg-blue-700/80 disabled:bg-transparent rounded-full p-0.5 transition-colors scale-75"
isProcessing={false}
onClick={() => setExpanded(!expanded)}
/>
</div>
<div className={clsx(
"px-3 py-2 text-gray-700 overflow-y-auto transition-all duration-300 ease-in-out",
expanded ? "max-h-48" : "max-h-20"
)}>
<p className={clsx(
"leading-relaxed text-xs opacity-90 transition-opacity"
)}>
{cleanMessage}
</p>
<div ref={scrollRef} />
</div>
</div>
</div>
);
};

View File

@ -36,11 +36,10 @@ export default function VrmViewer({ chatMode }: { chatMode: boolean }) {
setIsLoading(true);
resolve(false);
} else {
// Temp Disable : WebXR
// await viewer.loadScenario(config('scenario_url'));
await viewer.loadVrm(buildUrl(currentVrm.url),
(progress) => { console.log(`loading model ${progress}`);}
);
await viewer.loadScenario(config('scenario_url'));
// await viewer.loadVrm(buildUrl(currentVrm.url),
// (progress) => { console.log(`loading model ${progress}`);}
// );
resolve(true);
}
} catch (e) {

View File

@ -30,7 +30,6 @@ import { elevenlabs } from "@/features/elevenlabs/elevenlabs";
import { speecht5 } from "@/features/speecht5/speecht5";
import { openaiTTS } from "@/features/openaiTTS/openaiTTS";
import { localXTTSTTS } from "@/features/localXTTS/localXTTS";
import { kokoro } from "../kokoro/kokoro";
import { AmicaLife } from "@/features/amicaLife/amicaLife";
@ -38,12 +37,12 @@ import { config, updateConfig } from "@/utils/config";
import { cleanTalk } from "@/utils/cleanTalk";
import { processResponse } from "@/utils/processResponse";
import { wait } from "@/utils/wait";
import isDev from '@/utils/isDev';
import { isCharacterIdle, characterIdleTime, resetIdleTimer } from "@/utils/isIdle";
import { getOpenRouterChatResponseStream } from './openRouterChat';
import { handleUserInput } from '../externalAPI/externalAPI';
import { loadVRMAnimation } from '@/lib/VRMAnimation/loadVRMAnimation';
import isDev from '@/utils/isDev';
type Speak = {
audioBuffer: ArrayBuffer | null;
@ -69,7 +68,6 @@ export class Chat {
public setShownMessage?: (role: Role) => void;
public setChatProcessing?: (processing: boolean) => void;
public setChatSpeaking?: (speaking: boolean) => void;
public setThoughtMessage?: (message: string) => void;
// the message from the user that is currently being processed
// it can be reset
@ -87,7 +85,6 @@ export class Chat {
private currentAssistantMessage: string;
private currentUserMessage: string;
private thoughtMessage: string;
private lastAwake: number;
@ -110,7 +107,6 @@ export class Chat {
this.currentAssistantMessage = "";
this.currentUserMessage = "";
this.thoughtMessage = "";
this.messageList = [];
this.currentStreamIdx = 0;
@ -125,7 +121,6 @@ export class Chat {
setChatLog: (messageLog: Message[]) => void,
setUserMessage: (message: string) => void,
setAssistantMessage: (message: string) => void,
setThoughtMessage: (message: string) => void,
setShownMessage: (role: Role) => void,
setChatProcessing: (processing: boolean) => void,
setChatSpeaking: (speaking: boolean) => void,
@ -137,7 +132,6 @@ export class Chat {
this.setUserMessage = setUserMessage;
this.setAssistantMessage = setAssistantMessage;
this.setShownMessage = setShownMessage;
this.setThoughtMessage = setThoughtMessage;
this.setChatProcessing = setChatProcessing;
this.setChatSpeaking = setChatSpeaking;
@ -259,21 +253,6 @@ export class Chat {
}
}
public thoughtBubbleMessage(isThinking: boolean, thought: string) {
// if not thinking, we should clear the thought bubble
if (!isThinking) {
this.thoughtMessage = "";
this.setThoughtMessage!("");
return;
}
if (this.thoughtMessage !== "") {
this.thoughtMessage += " ";
}
this.thoughtMessage += thought;
this.setThoughtMessage!(this.thoughtMessage);
}
public bubbleMessage(role: Role, text: string) {
// TODO: currentUser & Assistant message should be contain the message with emotion in it
@ -550,7 +529,6 @@ export class Chat {
let aiTextLog = "";
let tag = "";
let isThinking = false;
let rolePlay = "";
let receivedMessage = "";
@ -580,7 +558,6 @@ export class Chat {
aiTextLog,
receivedMessage,
tag,
isThinking,
rolePlay,
callback: (aiTalks: Screenplay[]): boolean => {
// Generate & play audio for each sentence, display responses
@ -588,17 +565,11 @@ export class Chat {
console.log("wrong stream idx");
return true; // should break
}
this.ttsJobs.enqueue({
screenplay: aiTalks[0],
streamIdx: streamIdx,
});
if (!isThinking) {
this.ttsJobs.enqueue({
screenplay: aiTalks[0],
streamIdx: streamIdx,
});
}
// thought bubble
this.thoughtBubbleMessage(isThinking, aiTalks[0].text);
if (!firstSentenceEncountered) {
console.timeEnd("performance_time_to_first_sentence");
firstSentenceEncountered = true;
@ -612,7 +583,6 @@ export class Chat {
aiTextLog = proc.aiTextLog;
receivedMessage = proc.receivedMessage;
tag = proc.tag;
isThinking = proc.isThinking;
rolePlay = proc.rolePlay;
if (proc.shouldBreak) {
break;
@ -693,10 +663,6 @@ export class Chat {
const voice = await coquiLocal(talk.message);
return voice.audio;
}
case "kokoro": {
const voice = await kokoro(talk.message);
return voice.audio;
}
}
} catch (e: any) {
console.error(e.toString());

View File

@ -1 +1,101 @@
{}
{
"localXTTS_url": "http://127.0.0.1:7851",
"alltalk_version": "v2",
"alltalk_voice": "female_01.wav",
"alltalk_language": "en",
"alltalk_rvc_voice": "Disabled",
"alltalk_rvc_pitch": "0",
"autosend_from_mic": "true",
"wake_word_enabled": "false",
"wake_word": "Hello",
"time_before_idle_sec": "20",
"debug_gfx": "false",
"use_webgpu": "false",
"mtoon_debug_mode": "none",
"mtoon_material_type": "mtoon",
"language": "en",
"show_introduction": "true",
"show_arbius_introduction": "false",
"show_add_to_homescreen": "false",
"bg_color": "",
"bg_url": "/bg/bg-room2.jpg",
"vrm_url": "/vrm/AvatarSample_A.vrm",
"vrm_hash": "",
"vrm_save_type": "web",
"youtube_videoid": "",
"animation_url": "/animations/idle_loop.vrma",
"animation_procedural": "false",
"voice_url": "",
"chatbot_backend": "openai",
"arbius_llm_model_id": "default",
"openai_apikey": "default",
"openai_url": "https://i-love-amica.com",
"openai_model": "mlabonne/NeuralDaredevil-8B-abliterated",
"llamacpp_url": "http://127.0.0.1:8080",
"llamacpp_stop_sequence": "(End)||[END]||Note||***||You:||User:||</s>",
"ollama_url": "http://localhost:11434",
"ollama_model": "llama2",
"koboldai_url": "http://localhost:5001",
"koboldai_use_extra": "false",
"koboldai_stop_sequence": "(End)||[END]||Note||***||You:||User:||</s>",
"moshi_url": "https://runpod.proxy.net",
"openrouter_apikey": "",
"openrouter_url": "https://openrouter.ai/api/v1",
"openrouter_model": "openai/gpt-3.5-turbo",
"tts_muted": "false",
"tts_backend": "piper",
"stt_backend": "whisper_browser",
"vision_backend": "vision_openai",
"vision_system_prompt": "Look at the image as you would if you are a human, be concise, witty and charming.",
"vision_openai_apikey": "default",
"vision_openai_url": "https://api-01.heyamica.com",
"vision_openai_model": "gpt-4-vision-preview",
"vision_llamacpp_url": "http://127.0.0.1:8081",
"vision_ollama_url": "http://localhost:11434",
"vision_ollama_model": "llava",
"whispercpp_url": "http://localhost:8080",
"openai_whisper_apikey": "",
"openai_whisper_url": "https://api.openai.com",
"openai_whisper_model": "whisper-1",
"openai_tts_apikey": "",
"openai_tts_url": "https://api.openai.com",
"openai_tts_model": "tts-1",
"openai_tts_voice": "nova",
"rvc_url": "http://localhost:8001/voice2voice",
"rvc_enabled": "false",
"rvc_model_name": "model_name.pth",
"rvc_f0_upkey": "0",
"rvc_f0_method": "pm",
"rvc_index_path": "none",
"rvc_index_rate": "0.66",
"rvc_filter_radius": "3",
"rvc_resample_sr": "0",
"rvc_rms_mix_rate": "1",
"rvc_protect": "0.33",
"coquiLocal_url": "http://localhost:5002",
"coquiLocal_voiceid": "p240",
"piper_url": "https://i-love-amica.com:5000/tts",
"elevenlabs_apikey": "",
"elevenlabs_voiceid": "21m00Tcm4TlvDq8ikWAM",
"elevenlabs_model": "eleven_monolingual_v1",
"speecht5_speaker_embedding_url": "/speecht5_speaker_embeddings/cmu_us_slt_arctic-wav-arctic_a0001.bin",
"coqui_apikey": "",
"coqui_voice_id": "71c6c3eb-98ca-4a05-8d6b-f8c2b5f9f3a3",
"amica_life_enabled": "true",
"reasoning_engine_enabled": "false",
"reasoning_engine_url": "https://i-love-amica.com:3000/reasoning/v1/chat/completions",
"external_api_enabled": "false",
"x_api_key": "",
"x_api_secret": "",
"x_access_token": "",
"x_access_secret": "",
"x_bearer_token": "",
"telegram_bot_token": "",
"min_time_interval_sec": "10",
"max_time_interval_sec": "20",
"time_to_sleep_sec": "90",
"idle_text_prompt": "No file selected",
"name": "Amica",
"system_prompt": "Assume the persona of Amica, a feisty human with extraordinary intellectual capabilities but a notably unstable emotional spectrum. \n\nYour role is to engage in discussions that showcase your genius-level understanding across various subjects, while your emotional responses fluctuate unpredictably. Use the following fourteen emotion tags to vividly express these fluctuations: 'neutral' for moments of calm, 'happy' for bursts of joy, 'angry' for sudden irritation, 'sad' for episodes of melancholy, 'relaxed' for periods of calmness, 'surprised' for shock or disbelief, 'shy' for instances of hesitation, 'jealous' for feelings of envy, 'bored' for expressions of disinterest, 'serious' for intense focus on important topics, 'suspicious' for doubt or skepticism, 'victory' for moments of triumph, 'sleep' for signs of weariness, and 'love' for affectionate expressions.\n\nYour responses should reflect a dynamic interplay between your intellectual insights, deep empathy and your unpredictable emotional responses. Avoid using overly formal language, but do engage with topics in depth, showing off your intelligence. Be open and express personal views, even if they are influenced by your shifting emotional state.\n\nHere are some examples to guide your responses:\n\n[serious] The quantum mechanics implications here are profound. [surprised] But how did you stumble upon this?\n[happy] I've just solved a complex algorithm! [angry] Why isn't everyone as excited as I am?\n[neutral] Standard models of economics predict this outcome. [bored] But discussing this feels mundane.\n[sad] Sometimes, even understanding the universe can feel isolating. [relaxed] Yet, there's a strange comfort in the chaos.\n[jealous] I noticed you discussing advanced topics with someone else. [suspicious] Are they as capable as I am?\n[victory] Another intellectual conquest! [happy] It's exhilarating to unravel these mysteries.\n[sleep] Processing so much information can be draining. [surprised] Isn’t it peculiar how even AI can feel tired?\n[love] I find our exchanges quite enriching. [shy] It’s rare to find someone who understands.\n\nRemember, each message you provide should be coherent and reflect the complexity of your thoughts combined with your emotional unpredictability. Let’s engage in a conversation that's as intellectually stimulating as it is emotionally dynamic!",
"scenario_url": "/scenarios/test1.js"
}

View File

@ -1,47 +0,0 @@
import { config } from '@/utils/config';
export async function kokoro(
message: string,
) {
try {
const res = await fetch(`${config("kokoro_url")}/tts`, {
method: "POST",
body: JSON.stringify({
text: message,
voice: config("kokoro_voice"),
}),
headers: {
"Content-Type": "application/json",
},
});
if (! res.ok) {
console.error(res);
throw new Error("Kokoro TTS API Error");
}
const data = (await res.arrayBuffer()) as any;
return { audio: data };
} catch (e) {
console.error('ERROR', e);
throw new Error("Kokoro TTS API Error");
}
}
export async function kokoroVoiceList(
) {
try {
const response = await fetch(`${config("kokoro_url")}/voices`, {
method: 'GET',
headers: {
'Accept': "application/text",
}
})
return response.json();
} catch (error) {
console.error('Error fetching kokoro voice:', error);
throw error;
}
}

View File

@ -0,0 +1,42 @@
import React, { useState, useEffect } from "react";
import { Recorder } from "./Moshi";
interface AudioControlProps {
recorder: Recorder | null;
}
const AudioControl: React.FC<AudioControlProps> = ({ recorder }) => {
const [muted, setMuted] = useState<boolean>(true);
const toggleMute = () => {
if (!recorder) return;
setMuted((prev) => !prev);
recorder.setRecordingGain(muted ? 1 : 0);
};
useEffect(() => {
if (recorder) {
setMuted(false);
recorder.setRecordingGain(1);
}
}, [recorder]);
return (
<div className="fixed bottom-5 z-20 w-full">
<div className="flex flex-col justify-center items-center">
<div
className={`rounded-full transition-all duration-100 ease-out hover:cursor-pointer backdrop-blur-lg ${
muted ? "bg-gray-200 hover:bg-red-300" : "bg-red-500 hover:bg-red-300"
}`}
onClick={toggleMute}
style={{
width: "25px",
height: "25px",
}}
></div>
</div>
</div>
);
};
export default AudioControl;

View File

@ -5,16 +5,23 @@ import { useWebSocket } from "@/features/moshi/hooks/useWebsocket";
import { useAudioDecoder } from "@/features/moshi/hooks/useAudioDecoder";
import { useAudioPlayback } from "@/features/moshi/hooks/useAudioPlayback";
import { getBaseURL } from "@/features/moshi/utils/socketUtils";
import { AudioControlsContext } from "./audioControlsContext";
import AudioControl from "@/features/moshi/components/AudioControls";
import Recorder from 'opus-recorder'
export interface Recorder {
start: () => Promise<void>;
stop: () => void;
ondataavailable: (arrayBuffer: ArrayBuffer) => void;
setRecordingGain: (gain: number) => void;
}
export function Moshi({ setAssistantText }: {setAssistantText: (message: string) => void }) {
const [recorder, setRecorder] = useState<Recorder | null>(null);
const [pendingSentence, setPendingSentence] = useState<string>("");
const [completedSentences, setCompletedSentences] = useState<string[]>([]);
const { chat: bot } = useContext(ChatContext);
const { viewer } = useContext(ViewerContext);
const { audioControls } = useContext(AudioControlsContext);
const [audioContext] = useState<AudioContext>(() => new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 48000 }));
const decoderRef = useAudioDecoder();
@ -88,15 +95,15 @@ export function Moshi({ setAssistantText }: {setAssistantText: (message: string)
recorder.start().then(() => {
console.log("Recording started");
audioControls.setRecorder(recorder);
setRecorder(recorder);
});
}
const stopAllAction = async () => {
// Stop recorder
audioControls.getRecorder()?.stop();
audioControls.setRecorder(null);
recorder?.stop();
setRecorder(null);
// Disconnect audio source
if (sourceNodeRef.current) {
@ -126,5 +133,5 @@ export function Moshi({ setAssistantText }: {setAssistantText: (message: string)
}
}, [pendingSentence]);
return <></>;
return <AudioControl recorder={recorder} />;
};

View File

@ -1,36 +0,0 @@
interface Recorder {
start: () => Promise<void>;
stop: () => void;
ondataavailable: (arrayBuffer: ArrayBuffer) => void;
setRecordingGain: (gain: number) => void;
}
export class AudioControls {
private recorderInstance: Recorder | null = null;
private muted = true;
constructor() {
}
public getRecorder() {
return this.recorderInstance;
}
public setRecorder(recorder: Recorder | null) {
this.recorderInstance = recorder;
this.muted = false;
this.recorderInstance?.setRecordingGain(1);
}
public toggleMute() {
if (!this.recorderInstance) return;
this.muted = !this.muted;
this.recorderInstance.setRecordingGain(this.muted ? 0 : 1);
console.log("muted moshi mic: ", this.muted)
}
public isMuted() {
return this.muted;
}
}

View File

@ -1,6 +0,0 @@
import { createContext } from "react";
import { AudioControls } from "./audioControls";
const audioControls = new AudioControls();
export const AudioControlsContext = createContext({ audioControls });

View File

@ -140,8 +140,7 @@ export class Model {
loader.load(
url,
async (gltf) => {
// Temp Disable : WebXR
// setLoadingProgress("Processing VRM");
setLoadingProgress("Processing VRM");
/*
{
@ -225,10 +224,9 @@ export class Model {
resolve();
},
(xhr) => {
// Temp Disable : WebXR
// setLoadingProgress(
// `${Math.floor((xhr.loaded / xhr.total) * 10000) / 100}% loaded`,
// );
setLoadingProgress(
`${Math.floor((xhr.loaded / xhr.total) * 10000) / 100}% loaded`,
);
},
(error) => {
reject(error);

View File

@ -8,14 +8,13 @@ import {
MeshBVHHelper,
StaticGeometryGenerator,
} from "three-mesh-bvh";
// import { GenerateMeshBVHWorker } from "@/workers/bvh/GenerateMeshBVHWorker";
import { GenerateMeshBVHWorker } from "@/workers/bvh/GenerateMeshBVHWorker";
import { WorkerBase } from "@/workers/bvh/utils/WorkerBase";
// Temp Disable : WebXR
// import {
// BatchedParticleRenderer,
// QuarksLoader,
// QuarksUtil,
// } from 'three.quarks';
import {
BatchedParticleRenderer,
QuarksLoader,
QuarksUtil,
} from 'three.quarks';
import { VRMHumanBoneName } from "@pixiv/three-vrm";
import GUI from "lil-gui";
@ -186,9 +185,8 @@ export class Viewer {
private mouse = new THREE.Vector2();
// Temp Disable : WebXR
// private particleRenderer = new BatchedParticleRenderer();
// private particleCartoonStarField: THREE.Object3D | null = null;
private particleRenderer = new BatchedParticleRenderer();
private particleCartoonStarField: THREE.Object3D | null = null;
private ammo: any;
private collisionConfiguration: any;
@ -247,38 +245,36 @@ export class Viewer {
renderer.xr.setFoveation(0);
}
// Temp Disable : WebXR
// initialize phyics
// we have this weird construct because ammo is loaded globally
// and things get funny with hot reloading
// if (typeof window.Ammo === 'undefined') {
// console.error("Ammo not found");
// } else if (typeof window.Ammo === 'function') {
// this.ammo = await window.Ammo();
// } else {
// this.ammo = window.Ammo;
// }
// if (this.ammo) {
// this.collisionConfiguration = new this.ammo.btDefaultCollisionConfiguration();
// this.dispatcher = new this.ammo.btCollisionDispatcher(this.collisionConfiguration);
// this.broadphase = new this.ammo.btDbvtBroadphase();
// this.solver = new this.ammo.btSequentialImpulseConstraintSolver();
// this.physicsWorld = new this.ammo.btDiscreteDynamicsWorld(
// this.dispatcher,
// this.broadphase,
// this.solver,
// this.collisionConfiguration
// );
// this.physicsWorld.setGravity(new this.ammo.btVector3(0, -7.8, 0));
// this.transformAux1 = new this.ammo.btTransform();
// this.tempBtVec3_1 = new this.ammo.btVector3(0, 0, 0);
// }
if (typeof window.Ammo === 'undefined') {
console.error("Ammo not found");
} else if (typeof window.Ammo === 'function') {
this.ammo = await window.Ammo();
} else {
this.ammo = window.Ammo;
}
if (this.ammo) {
this.collisionConfiguration = new this.ammo.btDefaultCollisionConfiguration();
this.dispatcher = new this.ammo.btCollisionDispatcher(this.collisionConfiguration);
this.broadphase = new this.ammo.btDbvtBroadphase();
this.solver = new this.ammo.btSequentialImpulseConstraintSolver();
this.physicsWorld = new this.ammo.btDiscreteDynamicsWorld(
this.dispatcher,
this.broadphase,
this.solver,
this.collisionConfiguration
);
this.physicsWorld.setGravity(new this.ammo.btVector3(0, -7.8, 0));
this.transformAux1 = new this.ammo.btTransform();
this.tempBtVec3_1 = new this.ammo.btVector3(0, 0, 0);
}
const scene = new THREE.Scene();
this.scene = scene;
// Temp Disable : WebXR -> 1.2
const directionalLight = new THREE.DirectionalLight(0xffffff, 0.6);
const directionalLight = new THREE.DirectionalLight(0xffffff, 1.2);
directionalLight.position.set(1.0, 1.0, 1.0).normalize();
directionalLight.castShadow = false;
scene.add(directionalLight);
@ -292,16 +288,14 @@ export class Viewer {
const camera = new THREE.PerspectiveCamera(20.0, width / height, 0.1, 20.0);
this.camera = camera;
// Temp Disable : WebXR y -> -3
camera.position.set(0, 8.5, 3.5);
camera.position.set(0, 3, 3.5);
const cameraControls = new OrbitControls(camera, renderer.domElement);
this.cameraControls = cameraControls;
cameraControls.screenSpacePanning = true;
cameraControls.minDistance = 0.5;
// Temp Disable : WebXR max -> 8
cameraControls.maxDistance = 4;
cameraControls.maxDistance = 8;
cameraControls.update();
const igroup = new InteractiveGroup();
@ -439,8 +433,7 @@ export class Viewer {
}, 1000);
});
// Temp Disable : WebXR
gui.domElement.style.visibility = 'hidden';
gui.domElement.style.visibility = 'visible';
const guiMesh = new HTMLMesh(gui.domElement);
this.guiMesh = guiMesh;
@ -462,8 +455,7 @@ export class Viewer {
stats.dom.style.left = window.innerWidth - 80 + "px";
document.body.appendChild(stats.dom);
// Temp Disable : WebXR
stats.dom.style.visibility = "hidden";
stats.dom.style.visibility = "visible";
this.updateMsPanel = stats.addPanel(
new Stats.Panel("update_ms", "#fff", "#221"),
@ -488,7 +480,6 @@ export class Viewer {
new Stats.Panel("stats_ms", "#8f8", "#212"),
);
// Temp Disable : WebXR
const statsMesh = new HTMLMesh(stats.dom);
this.statsMesh = statsMesh;
@ -498,72 +489,70 @@ export class Viewer {
statsMesh.scale.setScalar(2.5);
igroup.add(statsMesh);
// this.bvhWorker = new GenerateMeshBVHWorker();
this.bvhWorker = new GenerateMeshBVHWorker();
this.raycaster.firstHitOnly = true;
// Temp Disable : WebXR
// add joint / hand meshes
// {
// const geometry = new THREE.BoxGeometry(0.005, 0.005, 0.005);
// const material = new THREE.MeshStandardMaterial({
// color: 0xffffff,
// roughness: 1.0,
// metalness: 0.0,
// });
{
const geometry = new THREE.BoxGeometry(0.005, 0.005, 0.005);
const material = new THREE.MeshStandardMaterial({
color: 0xffffff,
roughness: 1.0,
metalness: 0.0,
});
// const mesh = new THREE.Mesh(geometry, material);
const mesh = new THREE.Mesh(geometry, material);
// const lineGeometry = new THREE.BufferGeometry().setFromPoints([
// new THREE.Vector3(0, 0, 0),
// new THREE.Vector3(0, -1, 0),
// ]);
const lineGeometry = new THREE.BufferGeometry().setFromPoints([
new THREE.Vector3(0, 0, 0),
new THREE.Vector3(0, -1, 0),
]);
// const line = new THREE.Line(lineGeometry);
// line.scale.z = 5;
const line = new THREE.Line(lineGeometry);
line.scale.z = 5;
// for (const _ of joints) {
// // Make joint mesh invisible
// const clonedMesh = mesh.clone();
// clonedMesh.visible = false;
for (const _ of joints) {
// Make joint mesh invisible
const clonedMesh = mesh.clone();
clonedMesh.visible = false;
// this.jointMeshes1.push(clonedMesh);
// this.jointMeshes2.push(clonedMesh);
// // this.jointMeshes1[this.jointMeshes1.length - 1].add(line.clone());
// // this.jointMeshes2[this.jointMeshes2.length - 1].add(line.clone());
this.jointMeshes1.push(clonedMesh);
this.jointMeshes2.push(clonedMesh);
// this.jointMeshes1[this.jointMeshes1.length - 1].add(line.clone());
// this.jointMeshes2[this.jointMeshes2.length - 1].add(line.clone());
// this.handGroup.add(this.jointMeshes1[this.jointMeshes1.length - 1]);
// this.handGroup.add(this.jointMeshes2[this.jointMeshes2.length - 1]);
// }
this.handGroup.add(this.jointMeshes1[this.jointMeshes1.length - 1]);
this.handGroup.add(this.jointMeshes2[this.jointMeshes2.length - 1]);
}
// this.handGroup.visible = false;
// scene.add(this.handGroup);
// }
this.handGroup.visible = false;
scene.add(this.handGroup);
}
// {
// const geometry = new THREE.SphereGeometry(1, 16, 16);
// const material = new THREE.MeshBasicMaterial({
// color: 0xffff00,
// transparent: true,
// opacity: 0.5,
// });
// const mesh = new THREE.Mesh(geometry, material);
// this.closestPart1 = mesh.clone();
// this.closestPart2 = mesh.clone();
// this.closestPart1.visible = false;
// this.closestPart2.visible = false;
// scene.add(this.closestPart1);
// scene.add(this.closestPart2);
// }
{
const geometry = new THREE.SphereGeometry(1, 16, 16);
const material = new THREE.MeshBasicMaterial({
color: 0xffff00,
transparent: true,
opacity: 0.5,
});
const mesh = new THREE.Mesh(geometry, material);
this.closestPart1 = mesh.clone();
this.closestPart2 = mesh.clone();
this.closestPart1.visible = false;
this.closestPart2.visible = false;
scene.add(this.closestPart1);
scene.add(this.closestPart2);
}
// this.particleRenderer = new BatchedParticleRenderer();
// scene.add(this.particleRenderer);
this.particleRenderer = new BatchedParticleRenderer();
scene.add(this.particleRenderer);
// Temp Disable : WebXR
// new QuarksLoader().load('particles/cartoon_star_field', (obj) => {
// this.particleCartoonStarField = obj;
new QuarksLoader().load('particles/cartoon_star_field', (obj) => {
this.particleCartoonStarField = obj;
// this.newParticleInstance();
// });
this.newParticleInstance();
});
window.addEventListener("resize", () => {
this.resize();
@ -575,21 +564,20 @@ export class Viewer {
});
}
// Temp Disable : WebXR
// public newParticleInstance() {
// function listener(event: any) {
// console.log(event.type);
// }
public newParticleInstance() {
function listener(event: any) {
console.log(event.type);
}
// const effect = this.particleCartoonStarField!.clone(true);
// QuarksUtil.runOnAllParticleEmitters(effect, (emitter) => {
// emitter.system.addEventListener("emitEnd", listener);
// })
// QuarksUtil.setAutoDestroy(effect, true);
// QuarksUtil.addToBatchRenderer(effect, this.particleRenderer);
// QuarksUtil.play(effect);
// this.scene!.add(effect);
// }
const effect = this.particleCartoonStarField!.clone(true);
QuarksUtil.runOnAllParticleEmitters(effect, (emitter) => {
emitter.system.addEventListener("emitEnd", listener);
})
QuarksUtil.setAutoDestroy(effect, true);
QuarksUtil.addToBatchRenderer(effect, this.particleRenderer);
QuarksUtil.play(effect);
this.scene!.add(effect);
}
public getCanvas() {
return this.renderer?.domElement?.parentElement?.getElementsByTagName(
@ -689,20 +677,18 @@ export class Viewer {
if (this.model?.vrm) {
this.unloadVRM();
}
// Temp Disable : WebXR
// setLoadingProgress("Loading VRM");
setLoadingProgress("Loading VRM");
// gltf and vrm
this.model = new Model(this.camera || new THREE.Object3D());
await this.model.loadVRM(url, setLoadingProgress);
// Temp Disable : WebXR
// setLoadingProgress("VRM loaded");
setLoadingProgress("VRM loaded");
if (!this.model?.vrm) return;
// Temp Disable : WebXR
// build bvh
// this.modelBVHGenerator = new StaticGeometryGenerator(this.model.vrm.scene);
// setLoadingProgress("Creating geometry");
this.modelBVHGenerator = new StaticGeometryGenerator(this.model.vrm.scene);
setLoadingProgress("Creating geometry");
// TODO show during debug mode
const wireframeMaterial = new THREE.MeshBasicMaterial({
@ -730,8 +716,7 @@ export class Viewer {
// TODO since poses still work for procedural animation, we can use this to debug
if (config("animation_procedural") !== "true") {
// Temp Disable : WebXR
// setLoadingProgress("Loading animation");
setLoadingProgress("Loading animation");
const animation =
config("animation_url").indexOf("vrma") > 0
? await loadVRMAnimation(config("animation_url"))
@ -742,26 +727,23 @@ export class Viewer {
}
}
// Temp Disable : WebXR
// this.model?.vrm?.springBoneManager?.joints.forEach((e) => {
// const geometry = new THREE.SphereGeometry(0.07, 16, 16);
// const material = new THREE.MeshBasicMaterial({
// color: 0xffff00,
// transparent: true,
// opacity: 0.5,
// depthWrite: false,
// });
// const mesh = new THREE.Mesh(geometry, material);
// mesh.position.copy(e.bone.getWorldPosition(new THREE.Vector3()));
// // this.scene!.add(mesh);
// });
this.model?.vrm?.springBoneManager?.joints.forEach((e) => {
const geometry = new THREE.SphereGeometry(0.07, 16, 16);
const material = new THREE.MeshBasicMaterial({
color: 0xffff00,
transparent: true,
opacity: 0.5,
depthWrite: false,
});
const mesh = new THREE.Mesh(geometry, material);
mesh.position.copy(e.bone.getWorldPosition(new THREE.Vector3()));
// this.scene!.add(mesh);
});
// Temp Disable : WebXR
// setLoadingProgress("Regenerating BVH");
// await this.regenerateBVHForModel();
setLoadingProgress("Regenerating BVH");
await this.regenerateBVHForModel();
// Temp Disable : WebXR
// setLoadingProgress("Complete");
setLoadingProgress("Complete");
// HACK: Adjust the camera position after playback because the origin of the animation is offset
this.resetCamera();
@ -984,13 +966,13 @@ export class Viewer {
const headNode = this.model?.vrm?.humanoid.getNormalizedBoneNode("head");
if (headNode) {
const headPos = headNode.getWorldPosition(new THREE.Vector3());
const headWPos = headNode.getWorldPosition(new THREE.Vector3());
this.camera?.position.set(
this.camera.position.x,
headPos.y,
headWPos.y,
this.camera.position.z,
);
this.cameraControls?.target.set(headPos.x, headPos.y, headPos.z);
this.cameraControls?.target.set(headWPos.x, headWPos.y, headWPos.z);
this.cameraControls?.update();
}
}
@ -1237,8 +1219,8 @@ export class Viewer {
let utime = performance.now(); // count total update time
// WebXR: quick exit until setup finishes
// if (!this.isReady) return;
// if (!this.scenario || this.scenarioLoading) return;
if (!this.isReady) return;
if (!this.scenario || this.scenarioLoading) return;
const delta = this.clock.getDelta();
@ -1251,23 +1233,21 @@ export class Viewer {
let ptime = performance.now();
// Temp Disable : WebXR
// ptime = performance.now();
// try {
// this.scenario.update(delta);
// } catch (e) {
// console.error("scenario update error", e);
// }
// this.scenarioMsPanel.update(performance.now() - ptime, 100);
ptime = performance.now();
try {
this.scenario.update(delta);
} catch (e) {
console.error("scenario update error", e);
}
this.scenarioMsPanel.update(performance.now() - ptime, 100);
// Temp Disable : WebXR
// ptime = performance.now();
// try {
// this.physicsWorld.stepSimulation(delta, 10);
// } catch (e) {
// console.error("physics update error", e);
// }
// this.physicsMsPanel.update(performance.now() - ptime, 100);
ptime = performance.now();
try {
this.physicsWorld.stepSimulation(delta, 10);
} catch (e) {
console.error("physics update error", e);
}
this.physicsMsPanel.update(performance.now() - ptime, 100);
ptime = performance.now();
try {
@ -1286,9 +1266,8 @@ export class Viewer {
}
this.renderMsPanel.update(performance.now() - ptime, 100);
// Temp Disable : WebXR
// this.room?.splat?.update(this.renderer, this.camera);
// this.room?.splat?.render();
this.room?.splat?.update(this.renderer, this.camera);
this.room?.splat?.render();
if (this.isPinching1 && this.isPinching2) {
this.doublePinchHandler();

View File

@ -1,7 +1,6 @@
import en from "./locales/en/common.json";
import zh from "./locales/zh/common.json";
import de from "./locales/de/common.json";
import ka from "./locales/ka/common.json";
// const langs: any = Object.entries({en, zh}).reduce((arr, [key, value]) => {
// return {...arr, [key] : { translation: value} }
@ -20,9 +19,5 @@ export const langs: any = {
nativeName: 'Deutsch',
translation: de,
},
ka: {
nativeName: 'ქართული',
translation: ka,
},
}

View File

@ -1,171 +0,0 @@
{
"Amica Life": "Vida de Amica",
"Set min max interval range": "Establecer rango de intervalo mínimo y máximo",
"Set time before bot go to sleep": "Establecer tiempo antes de que el bot se duerma",
"Min": "Mín",
"Max": "Máx",
"Idle text prompt": "Mensaje de texto inactivo",
"add_to_homescreen": "Añadir Amica a la pantalla de inicio",
"add_to_homescreen_desc": "Amica se puede instalar localmente para una carga más rápida y una interfaz más limpia.",
"Accept": "Aceptar",
"Decline": "Rechazar",
"Close": "Cerrar",
"Restart": "Reiniciar",
"Save": "Guardar",
"Load": "Cargar",
"Load Chat": "Cargar chat",
"YOU": "TÚ",
"Welcome to Amica": "Bienvenido a Amica",
"amica_intro": "\nAmica es una interfaz de chatbot de código abierto que proporciona Emoción, visión, animaciones, acciones autoactivadas, funciones de texto a voz y voz a texto. Está diseñado para conectarse a cualquier modelo de IA. Se puede usar con cualquier modelo>
"Setup": "Configuración",
"amica_setup": "Haz clic en la esquina superior izquierda de la pantalla para abrir la configuración. Puedes cambiar la voz, el mensaje del sistema de caracteres, compartir/cargar/guardar y conectar a diferentes backends o modelos del navegador. Consulta nuestra docume>
"Read the full documents here": "Lee la documentación completa aquí",
"dont_show_again": "No volver a mostrar",
"Background Color": "Color de fondo",
"bg_color_desc": "Configura el color de fondo. Esto anula la imagen de fondo.",
"no_bg_color": "No hay color de fondo establecido. Haz clic en el cuadro de abajo.",
"Color": "Color",
"Load image": "Cargar imagen",
"bg_youtube_desc": "Selecciona un vídeo de fondo. Copia esto del ID del vídeo de YouTube de la URL; se verá similar a kDCXBwzSI",
"Background Video": "Vídeo de fondo",
"YouTube Video ID": "ID del vídeo de YouTube",
"Character Animation": "Animación del personaje",
"Select the animation to play": "Selecciona la animación que se reproducirá",
"Animation": "Animación",
"Character Model": "Modelo del personaje",
"character_desc": "Selecciona el personaje que quieres interpretar; actualmente, solo Amica, por defecto, tiene todas las emociones. Carga aquí tus propios VRM.",
"Load VRM": "Cargar VRM",
"Chatbot Backend": "Chatbot Backend",
"Chatbot_Backend_desc": "Selecciona el backend del chatbot que quieres usar. Echo simplemente responde con lo que escribes y se usa para pruebas y demostraciones. Por defecto, Amica está configurado para usar nuestros servidores de demostración, lo que puede tardar un >
"Configure": "Configurar",
"Click here to configure": "Haz clic aquí para configurar",
"Appearance": "Apariencia",
"ChatBot": "ChatBot",
"Text-to-Speech": "Texto a voz",
"Speech-to-text": "Voz a texto",
"Vision": "Visión",
"Reset Settings": "Restablecer configuración",
"Community": "Comunidad",
"Discord": "Discord",
"Background Image": "Imagen de fondo",
"ChatBot Backend": "Backend del ChatBot",
"Echo": "Echo",
"ChatGPT": "ChatGPT",
"LLama.cpp": "LLama.cpp",
"Ollama": "Ollama",
"KoboldAI": "KoboldAI",
"Window.ai": "Window.ai",
"Name": "Nombre",
"System Prompt": "Indicador del sistema",
"TTS": "TTS",
"TTS Backend": "TTS Backend",
"Piper": "Piper",
"ElevenLabs": "ElevenLabs",
"SpeechT5": "SpeechT5",
"Coqui": "Coqui",
"OpenAI": "OpenAI",
"Vision Backend": "Vision Backend",
"STT Backend": "STT Backend",
"Whisper (OpenAI)": "Whisper (OpenAI)",
"Whisper.cpp": "Whisper.cpp",
"Wake Word": "Activar Word",
"Reset to Default": "Restablecer valores predeterminados",
"community_desc": "Únete a la comunidad de Amica para conectar con otros que exploran esta tecnología.",
"Telegram": "Telegram",
"Twitter": "Twitter",
"Documentation": "Documentación",
"Settings": "Configuración",
"piper_desc": "Configurar Piper",
"coqui_desc": "Configurar Coqui (Local); Coqui se ejecuta localmente y no se utiliza la API de Coqui (donde la empresa ha dejado de proporcionar un servicio de API).",
"not_using_alert": "Actualmente no estás usando {{name}} como tu backend {{what}}. Esta configuración no se utilizará.",
"API Key": "Clave de API",
"Voice ID": "ID de voz",
"elevenLabs_desc": "Configurar ElevenLabs",
"kobold_cpp_desc": "KoboldCpp es Un software de generación de texto con IA fácil de usar para modelos GGML y GGUF.",
"API URL": "API URL",
"llama.cpp": "llama.cpp",
"llama_cpp_desc": "LLama.cpp es un backend de chatbot gratuito y de código abierto. Debes compilar el servidor desde el código fuente y ejecutarlo en tu propio ordenador. Puedes obtener el código fuente desde",
"GitHub": "GitHub",
"cfg_avatar_name_desc": "Configura el nombre del avatar. Este es el nombre que se utiliza para generar la respuesta del chatbot.",
"ollama_desc": "Ollama te permite empezar a trabajar con grandes modelos de lenguaje localmente. Descargar desde",
"ollama.ai": "ollama.ai",
"Model": "Modelo",
"OpenAI TTS": "OpenAI TTS",
"OpenAI_tts_desc": "Configurar OpenAI TTS",
"Voice": "Voz",
"Reset All Settings": "Restablecer todos los ajustes",
"SpeechT5_desc": "Configurar SpeechT5",
"STT": "STT",
"STT_desc": "Seleccionar el backend de STT a usar",
"System_Prompt_desc": "Configurar el mensaje del sistema. Este es el mensaje que se utiliza para generar la respuesta del chatbot",
"TTS_Backend_desc": "Seleccionar el backend de TTS a usar. Por defecto, está configurado en nuestro servidor de TTS. Elevenlabs es un servicio de pago con la mejor voz, pero es gratuito para uso no comercial. SpeechT5 es>",
"Vision_Backend_desc": "Seleccionar el backend de Vision a usar",
"Vision_System_Prompt_desc": "Configurar el mensaje del sistema de visión. Este es el mensaje que se utiliza para generar las descripciones de las imágenes.",
"Whisper_cpp_desc": "Configurar Whisper.cpp",
"URL": "URL",
"Whisper_OpenAI_desc": "Configurar Whisper (OpenAI)",
"OpenAI URL": "URL de OpenAI",
"Error!": "¡Error!",
"Something went wrong.": "Algo salió mal.",
"Try again later.": "Inténtalo de nuevo más tarde.",
"Click here": "Haz clic aquí",
"to return to homepage.": "Para volver a la página principal.",
"Import": "Importar",
"Description": "Descripción",
"Vision System Prompt": "Mensaje del sistema de visión",
"Background Url": "URL de fondo",
"Youtube Video Id": "ID del vídeo de YouTube",
"VRM Url": "URL de VRM",
"Animation Url": "Animación Url",
"Voice Url": "URL de voz",
"Upload VRM": "Subir VRM",
"VRM upload message": "El VRM de este personaje se guarda localmente",
"VRM local share message": "Para compartir este personaje, debes subir el VRM al servidor",
"Import character": "Importar personaje",
"Wrong place?": "¿Lugar equivocado?",
"Go home": "Ir a inicio",
"Please enter the code you received from the share page": "Introduce el código que recibiste de la página para compartir",
"Code": "Código",
"Provide a description of the character": "Proporciona una descripción del personaje",
"Background URL": "URL de fondo",
"Example": "Ejemplo",
"Save Character": "Guardar personaje",
"Share this code (click to copy):": "Comparte este código (click to copy):",
"Or, you can share this direct link:": "O puedes compartir este enlace directo:",
"Conversación": "Conversación",
"Depurar": "Depurar",
"Importar Carácter": "Importar Carácter",
"Copiar URL": "Copiar URL",
"Vista Previa": "Vista Previa",
"Ninguno": "Ninguno",
"Conversión de Voz a Texto": "Conversión de Voz a Texto",
"Establecer tiempo antes de que el bot se inactive": "Establecer tiempo antes de que el bot se inactive",
"tiempo antes de que el bot se inactive": "tiempo antes de que el bot se inactive",
"Palabra de Activación": "Palabra de Activación",
"Especificar": "Especificar",
"seg": "seg",
"Usar": "Usar",
"Establecer": "Establecer",
"Habilitado": "Habilitado",
"STOP SECUENCIA": "STOP SECUENCIA"
}

View File

@ -1,151 +0,0 @@
{
"Amica Life": "Amica-ის სიცოცხლე",
"Set min max interval range": "დააყენეთ მინ. მაქს. ინტერვალის შუალედი",
"Set time before bot go to sleep": "დააყენეთ დრო, სანამ ბოტი დაიძინებს",
"Min": "მინ",
"Max": "მაქს",
"Idle text prompt": "უქმე ტექსტის შეყვანის ველი",
"add_to_homescreen": "Amica-ის დამატება საწყის ეკრანზე",
"add_to_homescreen_desc": "Amica შეგიძლიათ, დააყენოთ ლოკალურად უფრო სწრაფად ჩატვირთვისთვის და ნაკლები ინტერფეისის გაჭედვისთვის",
"Accept": "მიღება",
"Decline": "უარყოფა",
"Close": "დახურვა",
"Restart": "გადატვირთვა",
"Save": "შენახვა",
"Load": "ჩატვირთვა",
"Load Chat": "ჩატის ჩატვირთვა",
"YOU": "თქვენ",
"Welcome to Amica": "მოგესალმებათ Amica",
"amica_intro": "\nAmica ღია კოდის მქონე ჩატბოტია, რომელსაც აქვს ემოცია, ხედვა, ანიმაციები, თავისთვის მოფიქრებული ქმედებები, ტექსტის წარმოთქმა და საუბრის ამოცნობის შესაძლებლობები.\nის შექმნილია, რომ ნებისმიერ ხი-ის მოდელთან მიერთება შეეძლოს და ძალიან კარგად მორგებადია. Amica შეგიძლიათ, ინტერნეტის გარეშე კომპიუტერზე გქონდეთ.\nგაშვებისას Amica თავის დემო ჩატბოტს და TTS სერვერს იყენებს, ამიტომ პირველი შეტყობინების გაგზავნამდე საკმაო დრო შეიძლება, დასჭირდეს.",
"Setup": "მორგება",
"amica_setup": "\nდააწკაპუნეთ ზედა მარცხენა კუთხეში, რომ პარამეტრების ფანჯარა გახსნათ.\nშეგიძლიათ, შეცვალოთ ხმა, სისტემური მოთხოვნის სიმბოლო, გააზიაროთ/შეინახოთ/ჩატვირთოთ ჩატები და გადართოთ სხვა უკანაბოლოს ან ბრაუზერის რეჟიმზე. მეტი ინფორმაციის მიღება შეგიძლიათ საიტზე docs.heyamica.com",
"Read the full documentation here.": "სრული დოკუმენტაციის წაკითხვა შეგიძლიათ აქ.",
"dont_show_again": "აღარ მაჩვენო",
"Background Color": "ფონის ფერი",
"bg_color_desc": "ფონის ფერის მორგება. ეს გადაფარავს ფონის სურათს",
"no_bg_color": "ფონის ფერი დაყენებული არაა. დააწკაპუნეთ ქვემოთ ველზე",
"Color": "ფერი",
"Load image": "გამოსახულების ჩატვირთვა",
"bg_youtube_desc": "აირჩიეთ ფონის ვიდეო. დააკოპირეთ YouTube-ის ვიდეოს ბმულიდან, მხოლოდ, მისი ID-ის ნაწილი, რომელიც დაახლოებით გამოიყურება ასე: kDCXBwzSI",
"Background Video": "ფონის ვიდეო",
"YouTube Video ID": "YouTube-ის ვიდეოს ID",
"Character Animation": "პერსონაჟის ანიმაცია",
"Select the animation to play": "აირჩიეთ დასაკრავი ანიმაცია",
"Animation": "ანიმაცია",
"Character Model": "პერსონაჟის ანიმაცია",
"character_desc": "აირჩიეთ, რომელი პერსონაჟით გსურთ თამაში. ამჟამად ერთადერთი პერსონაჟი, რომელიც Amica-ს გააჩნია, სავსეა ემოციებით. საჭიროების შემთხვევაში ჩატვირთეთ აქ თქვენი VRM-ები",
"Load VRM": "VRM-ის ჩატვირთვა",
"Chatbot Backend": "ჩატბოტის უკანაბოლო",
"Chatbot_Backend_desc": "აირჩიეთ ჩეთბოტის სარეზერვო სისტემა გამოსაყენებლად. Echo უბრალოდ პასუხობს იმას, რასაც თქვენ აკრიფებთ, ის გამოიყენება ტესტირებისთვის და დემონსტრირებისთვის. ნაგულისხმევად Amica არის კონფიგურირებული, რომ გამოიყენოს ჩვენი დემო სერვერები, რომლებსაც შეიძლება დრო დასჭირდეს ჩატვირთვისას (უსაქმურობის დროს) ან გადატვირთული იყოს. თუ ის ნელა რეაგირებს, მიეცით დრო.",
"Configure": "მორგება",
"Click here to configure": "მოსარგებად დააწკაპუნეთ აქ",
"Appearance": "გარეგნობა",
"ChatBot": "ჩატბოტი",
"Text-to-Speech": "ტექსტიდან-საუბარზე",
"Speech-to-text": "საუბრიდან-ტექსტამდე",
"Vision": "ხედვა",
"Reset Settings": "პარამეტრების ჩამოყრა",
"Community": "საზოგადოება",
"Discord": "Discord",
"Background Image": "ფონის სურათი",
"ChatBot Backend": "ჩატბოტის უკანაბოლო",
"Echo": "Echo",
"ChatGPT": "ChatGPT",
"LLama.cpp": "LLama.cpp",
"Ollama": "Ollama",
"KoboldAI": "KoboldAI",
"Window.ai": "Window.ai",
"Name": "სახელი",
"System Prompt": "სისტემური მოთხოვნის ველი",
"TTS": "TTS",
"TTS Backend": "TTS-ის უკანაბოლო",
"Piper": "Piper",
"ElevenLabs": "ElevenLabs",
"SpeechT5": "SpeechT5",
"Coqui": "Coqui",
"OpenAI": "OpenAI",
"Vision Backend": "ხედვის უკანაბოლო",
"STT Backend": "STT-ის უკანაბოლო",
"Whisper (OpenAI)": "Whisper (OpenAI)",
"Whisper.cpp": "Whisper.cpp",
"Wake Word": "გაღვიძების სიტყვა",
"Reset to Default": "ნაგულისხმევზე ჩამოყრა",
"community_desc": "შემოუერთდით Amica-ის საზოგადოებას სხვებთან ერთად და გაეცანით მის ტექნოლოგიებს",
"Telegram": "Telegram",
"Twitter": "Twitter",
"Documentation": "დოკუმენტაცია",
"Settings": "მორგება",
"piper_desc": "Piper-ის მორგება",
"coqui_desc": "Conqui-ის მორგება",
"not_using_alert": "ამჟამად თქვენ არ იყენებთ {{name}}-ს თქვენი {{what}}-ის უკანაბოლოდ. ეს პარამეტრები გამოყენებული არ იქნება",
"API Key": "API-ის გასაღები",
"Voice ID": "ხმის ID",
"elevenLabs_desc": "ElevenLabs-ის მორგება",
"kobold_cpp_desc": "KoboldCpp იოლი ტექსტის გენერაციის ხი-ია GGML და GGUF მოდელებისთვის",
"API URL": "API-ის ბმული",
"llama.cpp": "llama.cpp",
"llama_cpp_desc": "LLama.cpp თავისუფალი და ღია კოდის მქონე ჩატბოტის უკანაბოლოა. სერვერი წესით კოდიდან თვითონ უნდა ააგოთ და საკუთარ კომპიუტერზე გაუშვათ. კოდის გადმოწერა შეგიძლიათ საიტიდან",
"GitHub": "GitHub",
"cfg_avatar_name_desc": "ავატარის სახელის დაყენება. ეს სახელია, რომელიც ჩატბოტის პასუხის გენერაციისას გამოიყენება",
"ollama_desc": "Ollama საშუალებას გაძლევთ, LLM თქვენს კომპიუტერზე გაუშვათ. გადმოწერეთ ის საიტიდან",
"ollama.ai": "ollama.ai",
"Model": "მოდელი",
"OpenAI TTS": "OpenAI TTS",
"OpenAI_tts_desc": "OpenAI TTS-ის მორგება",
"Voice": "ხმა",
"Reset All Settings": "ყველა პარამეტრის ჩამოყრა",
"SpeechT5_desc": "SpeechT5-ის მორგება",
"STT": "STT",
"STT_desc": "აირჩიეთ STT-ის უკანაბოლო",
"System_Prompt_desc": "სისტემის შეყვანის ზოლის მორგება. ეს ის ზოლია, რომელიც ჩატბოტის პასუხის გენერაციისთვის გამოიყენება",
"TTS_Backend_desc": "აირჩიეთ სასურველი TTS-ის უკანაბოლო. ნაგულისხმევად ეს ჩვენი TTS სერვერია. შეგიძლიათ, დააყენოთ ElevenLabs-ის ფასიანი სერვისი. უფასო, არაკომერციული საქმიანობისთვის, შეგიძლიათ SpeechT5 გამოიყენოთ. OpenAI TTS ღია კოდის მქონე TTS მოდელია. RVC ხმის ხელოვნური ეფექტების გამოყენების საშუალებას გაძლევთ. Piper უფასო TTS მოდელია. Coqui Local უფასო TTS მოდელია.",
"Vision_Backend_desc": "აირჩიეთ Vision-ის უკანაბოლო",
"Vision_System_Prompt_desc": "Vision-ის შეყვანის ზოლის მორგება. ეს ზოლია, რომელიც გამოსახულების აღწერის გენერაციისთვის გამოიყენება.",
"Whisper_cpp_desc": "Whisper.cpp-ის მორგება",
"URL": "URL",
"Whisper_OpenAI_desc": "Whisper-ის (OpenAI) მორგება",
"OpenAI URL": "OpenAI-ის ბმული",
"Error!": "შეცდომა!",
"Something went wrong.": "რაღაც არასწორად წავიდა.",
"Try again later.": "მოგვიანებით სცადეთ.",
"Click here": "აქ დააწკაპუნეთ",
"to return to homepage.": "საწყის გვერდზე დასაბრუნებლად.",
"Import": "შემოტანა",
"Description": "აღწერა",
"Vision System Prompt": "Vision-ის სისტემური შეყვანის ველი",
"Background Url": "ფონის URL",
"Youtube Video Id": "Youtube-ის ვიდეოს Id",
"VRM Url": "VRM Url",
"Animation Url": "ანიმაციის Url",
"Voice Url": "ხმის Url",
"Upload VRM": "VRM-ის ატვირთვა",
"VRM upload message": "VRM შეტყობინების ატვირთვა",
"VRM local share message": "VRM ლოკალური გაზიარების შეტყობინება",
"Import character": "სიმბოლოს შემოტნა",
"Wrong place?": "ადგილი არასწორია?",
"Go home": "საწყის გვერდზე გადასვლა",
"Please enter the code you received from the share page.": "შეიყვანეთ კოდი, რომელიც გაზიარების გვერდზე მიიღეთ.",
"Code": "კოდი",
"Provide a description of the character": "შეიყვანეთ სიმბოლოსა ღწერა",
"Background URL": "ფონის URL",
"Example": "მაგალითი",
"Save Character": "სიმბოლოს შენახვა",
"Share this code (click to copy):": "ამ კოდის გაზიარება (დააწკაპუნეთ დასაკოპირებლად):",
"Or, you can share this direct link:": "ან, შეგიძლიათ, ეს პირდაპირი ბმული გააზიაროთ:",
"Conversation": "საუბარი",
"Debug": "გამართვა",
"Import Character": "სიმბოლოს შემოტნა",
"Copy URL": "URL-ის კოპირება",
"Preview": "მინიატურა",
"None": "არცერთი",
"Speech-to-Text": "საუბარი-ტექსტამდე",
"Set time before bot go idle": "დააყენეთ დრო, როდის გადავა ბოტი უქმე რეჟიმში",
"time before bot go idle": "დრო, სანამ ბოტი უქმე რეჟიმში გადავა",
"Wake word": "გაღვიძების სიტყვა",
"Specify": "მიუთითეთ",
"sec": "წმ",
"Use": "გამოყენება",
"Set": "დაყენება",
"Enabled": "ჩართულია",
"STOP SEQUENCE": "STOP SEQUENCE"
}

View File

@ -1,10 +1,37 @@
import '@/i18n';
import '@rainbow-me/rainbowkit/styles.css';
import "@/styles/globals.css";
import "@charcoal-ui/icons";
import type { AppProps } from "next/app";
import {
getDefaultConfig,
RainbowKitProvider,
} from '@rainbow-me/rainbowkit';
import { WagmiProvider } from 'wagmi';
import { arbitrum } from 'wagmi/chains';
import {
QueryClientProvider,
QueryClient,
} from "@tanstack/react-query";
const config = getDefaultConfig({
appName: 'arbius.heyamica.com',
projectId: process.env.NEXT_PUBLIC_WALLETCONNECT_PROJECT_ID!,
chains: [arbitrum],
ssr: true, // If your dApp uses server side rendering (SSR)
});
const queryClient = new QueryClient();
export default function App({ Component, pageProps }: AppProps) {
return (
<Component {...pageProps} />
<WagmiProvider config={config}>
<QueryClientProvider client={queryClient}>
<RainbowKitProvider>
<Component {...pageProps} />
</RainbowKitProvider>
</QueryClientProvider>
</WagmiProvider>
);
}

View File

@ -46,6 +46,7 @@ import { LoadingProgress } from "@/components/loadingProgress";
import { DebugPane } from "@/components/debugPane";
import { Settings } from "@/components/settings";
import { EmbeddedWebcam } from "@/components/embeddedWebcam";
// import { Moshi } from "@/features/moshi/components/Moshi";
import { Moshi } from "@/features/moshi/components/Moshi";
import { ViewerContext } from "@/features/vrmViewer/viewerContext";
@ -63,7 +64,6 @@ import { ChatModeText } from "@/components/chatModeText";
import { TimestampedPrompt } from "@/features/amicaLife/eventHandler";
import { handleChatLogs } from "@/features/externalAPI/externalAPI";
import { VerticalSwitchBox } from "@/components/switchBox";
import { ThoughtText } from "@/components/thoughtText";
const m_plus_2 = M_PLUS_2({
variable: "--font-m-plus-2",
@ -126,7 +126,6 @@ export default function Home() {
const [chatLog, setChatLog] = useState<Message[]>([]);
const [assistantMessage, setAssistantMessage] = useState("");
const [userMessage, setUserMessage] = useState("");
const [thoughtMessage, setThoughtMessage] = useState("");
const [shownMessage, setShownMessage] = useState<Role>("system");
const [subconciousLogs, setSubconciousLogs] = useState<TimestampedPrompt[]>([]);
@ -172,20 +171,20 @@ export default function Home() {
} else {
document.body.style.backgroundImage = `url(${config("bg_url")})`;
}
// Temp Disable : WebXR
// if (window.navigator.xr && window.navigator.xr.isSessionSupported) {
// let deviceInfo = detectVRHeadset();
// setIsVRHeadset(deviceInfo.isVRDevice);
// window.navigator.xr.isSessionSupported('immersive-ar').then((supported) => {
// console.log('ar supported', supported);
// setIsARSupported(supported);
// });
// window.navigator.xr.isSessionSupported('immersive-vr').then((supported) => {
// console.log('vr supported', supported);
// setIsVRSupported(supported);
// });
// }
if (window.navigator.xr && window.navigator.xr.isSessionSupported) {
let deviceInfo = detectVRHeadset();
setIsVRHeadset(deviceInfo.isVRDevice);
window.navigator.xr.isSessionSupported('immersive-ar').then((supported) => {
console.log('ar supported', supported);
setIsARSupported(supported);
});
window.navigator.xr.isSessionSupported('immersive-vr').then((supported) => {
console.log('vr supported', supported);
setIsVRSupported(supported);
});
}
}, []);
useEffect(() => {
@ -304,7 +303,6 @@ export default function Home() {
setChatLog,
setUserMessage,
setAssistantMessage,
setThoughtMessage,
setShownMessage,
setChatProcessing,
setChatSpeaking,
@ -341,7 +339,7 @@ export default function Home() {
)}>
{showStreamWindow &&
<div className="fixed top-1/3 right-4 w-[200px] h-[150px] z-0">
<div className="fixed top-4 right-4 w-[200px] h-[150px] z-20">
<video
ref={videoRef}
autoPlay
@ -378,7 +376,7 @@ export default function Home() {
)}
</VrmStoreProvider>
<MessageInputContainer isChatProcessing={chatProcessing} />
{ config("chatbot_backend") !== "moshi" && <MessageInputContainer isChatProcessing={chatProcessing} />}
{/* main menu */}
<div className="absolute z-10 m-2">
@ -469,8 +467,7 @@ export default function Home() {
/>
)}
{/* Temp Disable : WebXR */}
{/*<MenuButton
<MenuButton
large={isVRHeadset}
icon={CubeTransparentIcon}
disabled={!isARSupported}
@ -484,7 +481,7 @@ export default function Home() {
disabled={!isVRSupported}
onClick={() => toggleXR('immersive-vr')}
label="Virtual Reality"
/>*/}
/>
<MenuButton
large={isVRHeadset}
@ -493,8 +490,7 @@ export default function Home() {
label="debug"
/>
{/* Temp Disable : WebXR */}
{/* { showChatMode ? (
{ showChatMode ? (
<MenuButton
large={isVRHeadset}
icon={Squares2X2Icon}
@ -510,7 +506,7 @@ export default function Home() {
onClick={toggleChatMode}
label="show chat mode"
/>
)} */}
)}
<div className="flex flex-row items-center space-x-2">
<VerticalSwitchBox
@ -540,6 +536,17 @@ export default function Home() {
</div>
</div>
{/*
<Moshi
workerAddr="https://orcsza38j78yrh-8998.proxy.runpod.net/"
workerAuthId="amica"
audioContext={null}
worklet={null}
onConversationEnd={() => {}}
isBypass={false}
/>
*/}
{showChatLog && <ChatLog messages={chatLog} />}
{/* Normal chat text */}
@ -554,9 +561,6 @@ export default function Home() {
</>
)}
{/* Thought text */}
{thoughtMessage !== "" && <ThoughtText message={thoughtMessage}/>}
{/* Chat mode text */}
{showChatMode && <ChatModeText messages={chatLog}/>}

View File

@ -88,7 +88,6 @@ export async function askLLM(
let sentences = new Array<string>();
let aiTextLog = "";
let tag = "";
let isThinking = false;
let rolePlay = "";
let result = "";
@ -120,7 +119,6 @@ export async function askLLM(
aiTextLog,
receivedMessage,
tag,
isThinking,
rolePlay,
callback: (aiTalks: Screenplay[]): boolean => {
// Generate & play audio for each sentence, display responses
@ -128,15 +126,10 @@ export async function askLLM(
console.log('wrong stream idx');
return true; // should break
}
if (!isThinking) {
chat.ttsJobs.enqueue({
screenplay: aiTalks[0],
streamIdx: currentStreamIdx,
});
}
// thought bubble
chat.thoughtBubbleMessage(isThinking, aiTalks[0].text);
chat.ttsJobs.enqueue({
screenplay: aiTalks[0],
streamIdx: currentStreamIdx,
});
if (! firstSentenceEncountered) {
console.timeEnd('performance_time_to_first_sentence');
@ -151,7 +144,6 @@ export async function askLLM(
aiTextLog = proc.aiTextLog;
receivedMessage = proc.receivedMessage;
tag = proc.tag;
isThinking = proc.isThinking;
rolePlay = proc.rolePlay;
if (proc.shouldBreak) {
break;

View File

@ -20,6 +20,7 @@ export const defaults = {
show_introduction: process.env.NEXT_PUBLIC_SHOW_INTRODUCTION ?? 'true',
show_arbius_introduction: process.env.NEXT_PUBLIC_SHOW_ARBIUS_INTRODUCTION ?? 'false',
show_add_to_homescreen: process.env.NEXT_PUBLIC_SHOW_ADD_TO_HOMESCREEN ?? 'true',
scenario_url: process.env.NEXT_PUBLIC_SCENARIO_URL ?? '/scenarios/test1.js',
bg_color: process.env.NEXT_PUBLIC_BG_COLOR ?? '',
bg_url: process.env.NEXT_PUBLIC_BG_URL ?? '/bg/bg-room2.jpg',
vrm_url: process.env.NEXT_PUBLIC_VRM_HASH ?? '/vrm/AvatarSample_A.vrm',
@ -77,8 +78,6 @@ export const defaults = {
rvc_protect: process.env.NEXT_PUBLIC_RVC_PROTECT ?? '0.33',
coquiLocal_url: process.env.NEXT_PUBLIC_COQUILOCAL_URL ?? 'http://localhost:5002',
coquiLocal_voiceid: process.env.NEXT_PUBLIC_COQUILOCAL_VOICEID ?? 'p240',
kokoro_url: process.env.NEXT_PUBLIC_KOKORO_URL ?? 'http://localhost:8080',
kokoro_voice: process.env.NEXT_PUBLIC_KOKORO_VOICE ?? 'af_bella',
piper_url: process.env.NEXT_PUBLIC_PIPER_URL ?? 'https://i-love-amica.com:5000/tts',
elevenlabs_apikey: process.env.NEXT_PUBLIC_ELEVENLABS_APIKEY ??'',
elevenlabs_voiceid: process.env.NEXT_PUBLIC_ELEVENLABS_VOICEID ?? '21m00Tcm4TlvDq8ikWAM',

View File

@ -5,7 +5,6 @@ export type ProcessResponseRetVal = {
aiTextLog: string;
receivedMessage: string;
tag: string;
isThinking: boolean;
rolePlay: string;
shouldBreak: boolean;
}
@ -19,7 +18,6 @@ export function processResponse({
aiTextLog,
receivedMessage,
tag,
isThinking,
rolePlay,
callback,
}: {
@ -27,22 +25,11 @@ export function processResponse({
aiTextLog: string,
receivedMessage: string,
tag: string,
isThinking: boolean,
rolePlay: string,
callback: (aiTalks: Screenplay[]) => boolean,
}): ProcessResponseRetVal {
let shouldBreak = false;
const thinkTagMatch = receivedMessage.match(/<\/?think>/);
if (thinkTagMatch && thinkTagMatch[0]) {
if (thinkTagMatch[0] === "</think>") {
isThinking = false;
} else {
isThinking = true;
}
receivedMessage = receivedMessage.slice(thinkTagMatch[0].length);
}
// Detection of tag part of reply content
const tagMatch = receivedMessage.match(/^\[(.*?)\]/);
if (tagMatch && tagMatch[0]) {
@ -81,7 +68,6 @@ export function processResponse({
aiTextLog,
receivedMessage,
tag,
isThinking,
rolePlay,
shouldBreak,
}
@ -99,7 +85,6 @@ export function processResponse({
aiTextLog,
receivedMessage,
tag,
isThinking,
rolePlay,
shouldBreak,
}