diff --git a/package-lock.json b/package-lock.json index 0444176..d00f5ff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "lala-companion", - "version": "0.0.10", + "version": "0.0.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "lala-companion", - "version": "0.0.10", + "version": "0.0.11", "license": "AGPL-3.0", "dependencies": { "@emotion/react": "^11.11.3", @@ -17,6 +17,7 @@ "@pixiv/three-vrm": "^2.0.10", "@react-three/drei": "^9.97.3", "@react-three/fiber": "^8.15.16", + "@react-three/rapier": "^1.2.1", "ai": "^2.2.34", "dotenv": "^16.4.1", "electron-squirrel-startup": "^1.0.0", @@ -292,6 +293,35 @@ "node": ">=6.9.0" } }, + "node_modules/@chevrotain/cst-dts-gen": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/@chevrotain/cst-dts-gen/-/cst-dts-gen-10.5.0.tgz", + "integrity": "sha512-lhmC/FyqQ2o7pGK4Om+hzuDrm9rhFYIJ/AXoQBeongmn870Xeb0L6oGEiuR8nohFNL5sMaQEJWCxr1oIVIVXrw==", + "dependencies": { + "@chevrotain/gast": "10.5.0", + "@chevrotain/types": "10.5.0", + "lodash": "4.17.21" + } + }, + "node_modules/@chevrotain/gast": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/@chevrotain/gast/-/gast-10.5.0.tgz", + "integrity": "sha512-pXdMJ9XeDAbgOWKuD1Fldz4ieCs6+nLNmyVhe2gZVqoO7v8HXuHYs5OV2EzUtbuai37TlOAQHrTDvxMnvMJz3A==", + "dependencies": { + "@chevrotain/types": "10.5.0", + "lodash": "4.17.21" + } + }, + "node_modules/@chevrotain/types": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/@chevrotain/types/-/types-10.5.0.tgz", + "integrity": "sha512-f1MAia0x/pAVPWH/T73BJVyO2XU5tI4/iE7cnxb7tqdNTNhQI3Uq3XkqcoteTmD4t1aM0LbHCJOhgIDn07kl2A==" + }, + "node_modules/@chevrotain/utils": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/@chevrotain/utils/-/utils-10.5.0.tgz", + "integrity": "sha512-hBzuU5+JjB2cqNZyszkDHZgOSrUUT8V3dhgRl8Q9Gp6dAj/H5+KILGjbhDpc3Iy9qmqlm/akuOI2ut9VUtzJxQ==" + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -314,6 +344,11 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@dimforge/rapier3d-compat": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/@dimforge/rapier3d-compat/-/rapier3d-compat-0.11.2.tgz", + "integrity": "sha512-vdWmlkpS3G8nGAzLuK7GYTpNdrkn/0NKCe0l1Jqxc7ZZOB3N0q9uG/Ap9l9bothWuAvxscIt0U97GVLr0lXWLg==" + }, "node_modules/@electron-forge/cli": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@electron-forge/cli/-/cli-7.2.0.tgz", @@ -2705,6 +2740,42 @@ } } }, + "node_modules/@react-three/rapier": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@react-three/rapier/-/rapier-1.2.1.tgz", + "integrity": "sha512-h22uqwvMBVVmwfe37fs0tF3u+gNDJv/LzS4lRYnZ4IaHZPlprAjvccef6R7pFlW2EU/o26HhzTMoE5/Btkdcew==", + "dependencies": { + "@dimforge/rapier3d-compat": "0.11.2", + "three-stdlib": "2.23.9", + "use-asset": "1.0.4" + }, + "peerDependencies": { + "@react-three/fiber": ">=8.9.0", + "react": ">=18.0.0", + "three": ">=0.139.0" + } + }, + "node_modules/@react-three/rapier/node_modules/three-stdlib": { + "version": "2.23.9", + "resolved": "https://registry.npmjs.org/three-stdlib/-/three-stdlib-2.23.9.tgz", + "integrity": "sha512-fYBClVGQptD7UZcoRZGNlR3sKcUW37hVPoEW1v68E4XuiwD0Ml/VqDUJ0yEMVE2DlooDvqgqv/rIcHC/B4N5pg==", + "dependencies": { + "@types/draco3d": "^1.4.0", + "@types/offscreencanvas": "^2019.6.4", + "@types/webxr": "^0.5.2", + "chevrotain": "^10.1.2", + "draco3d": "^1.4.1", + "fflate": "^0.6.9", + "ktx-parse": "^0.4.5", + "mmd-parser": "^1.0.4", + "opentype.js": "^1.3.3", + "potpack": "^1.0.1", + "zstddec": "^0.0.2" + }, + "peerDependencies": { + "three": ">=0.128.0" + } + }, "node_modules/@sindresorhus/is": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", @@ -4754,6 +4825,19 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chevrotain": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-10.5.0.tgz", + "integrity": "sha512-Pkv5rBY3+CsHOYfV5g/Vs5JY9WTHHDEKOlohI2XeygaZhUeqhAlldZ8Hz9cRmxu709bvS08YzxHdTPHhffc13A==", + "dependencies": { + "@chevrotain/cst-dts-gen": "10.5.0", + "@chevrotain/gast": "10.5.0", + "@chevrotain/types": "10.5.0", + "@chevrotain/utils": "10.5.0", + "lodash": "4.17.21", + "regexp-to-ast": "0.5.0" + } + }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -7006,8 +7090,7 @@ "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "node_modules/fast-glob": { "version": "3.3.2", @@ -9001,6 +9084,11 @@ "node": ">=0.10.0" } }, + "node_modules/ktx-parse": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/ktx-parse/-/ktx-parse-0.4.5.tgz", + "integrity": "sha512-MK3FOody4TXbFf8Yqv7EBbySw7aPvEcPX++Ipt6Sox+/YMFvR5xaTyhfNSk1AEmMy+RYIw81ctN4IMxCB8OAlg==" + }, "node_modules/launch-editor": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.6.1.tgz", @@ -9145,8 +9233,7 @@ "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/lodash._reinterpolate": { "version": "3.0.0", @@ -9653,6 +9740,11 @@ "mkdirp": "bin/cmd.js" } }, + "node_modules/mmd-parser": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mmd-parser/-/mmd-parser-1.0.4.tgz", + "integrity": "sha512-Qi0VCU46t2IwfGv5KF0+D/t9cizcDug7qnNoy9Ggk7aucp0tssV8IwTMkBlDbm+VqAf3cdQHTCARKSsuS2MYFg==" + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -10116,6 +10208,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/opentype.js": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/opentype.js/-/opentype.js-1.3.4.tgz", + "integrity": "sha512-d2JE9RP/6uagpQAVtJoF0pJJA/fgai89Cc50Yp0EJHk+eLp6QQ7gBoblsnubRULNY132I0J1QKMJ+JTbMqz4sw==", + "dependencies": { + "string.prototype.codepointat": "^0.2.1", + "tiny-inflate": "^1.0.3" + }, + "bin": { + "ot": "bin/ot" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/optionator": { "version": "0.9.3", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", @@ -11199,6 +11306,11 @@ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" }, + "node_modules/regexp-to-ast": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/regexp-to-ast/-/regexp-to-ast-0.5.0.tgz", + "integrity": "sha512-tlbJqcMHnPKI9zSrystikWKwHkBqu2a/Sgw01h3zFjvYrMxEDYHzzoMZnUrbIfpTFEsoRnnviOXNCzFiSc54Qw==" + }, "node_modules/regexp.prototype.flags": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz", @@ -12151,6 +12263,11 @@ "node": ">=8" } }, + "node_modules/string.prototype.codepointat": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/string.prototype.codepointat/-/string.prototype.codepointat-0.2.1.tgz", + "integrity": "sha512-2cBVCj6I4IOvEnjgO/hWqXjqBGsY+zwPmHl12Srk9IXSZ56Jwwmy+66XO5Iut/oQVR7t5ihYdLB0GMa4alEUcg==" + }, "node_modules/string.prototype.trim": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz", @@ -12604,6 +12721,11 @@ "dev": true, "optional": true }, + "node_modules/tiny-inflate": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", + "integrity": "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==" + }, "node_modules/tinycolor2": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", @@ -13113,6 +13235,17 @@ "punycode": "^2.1.0" } }, + "node_modules/use-asset": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/use-asset/-/use-asset-1.0.4.tgz", + "integrity": "sha512-7/hqDrWa0iMnCoET9W1T07EmD4Eg/Wmoj/X8TGBc++ECRK4m5yTsjP4O6s0yagbxfqIOuUkIxe2/sA+VR2GxZA==", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "react": ">=17.0" + } + }, "node_modules/use-sync-external-store": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz", @@ -13995,6 +14128,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zstddec": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/zstddec/-/zstddec-0.0.2.tgz", + "integrity": "sha512-DCo0oxvcvOTGP/f5FA6tz2Z6wF+FIcEApSTu0zV5sQgn9hoT5lZ9YRAKUraxt9oP7l4e8TnNdi8IZTCX6WCkwA==" + }, "node_modules/zustand": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/zustand/-/zustand-3.7.2.tgz", diff --git a/package.json b/package.json index b801ef4..fc9b1e0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "lala-companion", "productName": "lala-companion", - "version": "0.0.10", + "version": "0.0.11", "description": "3D personified desktop assistants, tuned for you, powered by AI vision and voice.", "main": ".webpack/main", "scripts": { @@ -55,6 +55,7 @@ "@pixiv/three-vrm": "^2.0.10", "@react-three/drei": "^9.97.3", "@react-three/fiber": "^8.15.16", + "@react-three/rapier": "^1.2.1", "ai": "^2.2.34", "dotenv": "^16.4.1", "electron-squirrel-startup": "^1.0.0", diff --git a/src/Scene.tsx b/src/Scene.tsx new file mode 100644 index 0000000..d2f4310 --- /dev/null +++ b/src/Scene.tsx @@ -0,0 +1,84 @@ +import { VRM } from "@pixiv/three-vrm"; +import { OrbitControls } from "@react-three/drei"; +import { Canvas } from "@react-three/fiber"; +import { RapierRigidBody } from "@react-three/rapier"; +import React, { useRef, useEffect, MutableRefObject } from "react"; +import { Mesh } from "three"; +import { animations } from "./constants/animations"; +import VrmCompanion from "./components/VRMCompanion"; + +interface SceneProps { + virtualText: string; + voiceUrl: string; + audioRef?: MutableRefObject; + onSpeakStart?: () => void; + onSpeakEnd?: () => void; +} + +const Scene = ({ + virtualText, + voiceUrl, + onSpeakStart, + onSpeakEnd, +}: SceneProps) => { + const vrmRef = useRef(null); + const vrmMeshRef = useRef(null); + const vrmPhysicsRef = useRef(null); + + useEffect(() => { + if (virtualText) { + (vrmRef as any)?.current?.setText?.(virtualText); + } + }, [virtualText]); + + useEffect(() => { + const speak = async () => { + if (voiceUrl) { + onSpeakStart?.(); + await (vrmRef as any)?.current?.talk?.(voiceUrl); + onSpeakEnd?.(); + } + }; + speak(); + }, [voiceUrl]); + + return ( + <> + + + + + {/* left */} + + + {/* right */} + + + + + + ); +}; + +export default Scene; diff --git a/src/components/VRMCompanion.tsx b/src/components/VRMCompanion.tsx index 81ea87e..76d180a 100644 --- a/src/components/VRMCompanion.tsx +++ b/src/components/VRMCompanion.tsx @@ -1,313 +1,500 @@ import React, { MutableRefObject, Suspense, + forwardRef, useCallback, useEffect, + useImperativeHandle, useMemo, useRef, useState, } from "react"; -import { Canvas, useFrame } from "@react-three/fiber"; -import { OrbitControls, Text } from "@react-three/drei"; -import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader"; -import { VRM, VRMUtils, VRMLoaderPlugin } from "@pixiv/three-vrm"; +import { useFrame } from "@react-three/fiber"; import { + GLTF, + GLTFLoader, + GLTFParser, +} from "three/examples/jsm/loaders/GLTFLoader"; +import { + VRM, + VRMUtils, + VRMLoaderPlugin, + VRMSpringBoneColliderShapeCapsule, + VRMSpringBoneColliderShapeSphere, + VRMExpressionPresetName, +} from "@pixiv/three-vrm"; +import { + AnimationAction, AnimationClip, AnimationMixer, + Euler, LoopOnce, + Mesh, NumberKeyframeTrack, + Vector3, } from "three"; import { loadMixamoAnimation } from "../helpers/loadMixamoAnimation"; +import { RapierRigidBody, RigidBody } from "@react-three/rapier"; +import { Text } from "@react-three/drei"; + +export const emotions = { + happy: VRMExpressionPresetName.Happy, + sad: VRMExpressionPresetName.Sad, + angry: VRMExpressionPresetName.Angry, + relaxed: VRMExpressionPresetName.Relaxed, + surprised: VRMExpressionPresetName.Surprised, + neutral: VRMExpressionPresetName.Neutral, +}; -const vrms = [ - { - rotation: [0.1, 3, 0], - model: "https://lalaland.chat/vrms/purple-girl.vrm", - scale: [1, 1, 1], - animations: { - idle: [ - "https://lalaland.chat/animations/idle/idle-1.fbx", - "https://lalaland.chat/animations/idle/idle-2.fbx", - "https://lalaland.chat/animations/idle/idle-3.fbx", - "https://lalaland.chat/animations/idle/idle-4.fbx", - ], - greet: [ - "https://lalaland.chat/animations/greet/greet-1.fbx", - "https://lalaland.chat/animations/bored/female-pose-standing.fbx", - "https://lalaland.chat/animations/bored/rumba-dance.fbx", - "https://lalaland.chat/animations/bored/ymca.fbx", - "https://lalaland.chat/animations/bored/yawn.fbx", - ], - talk: [ - "https://lalaland.chat/animations/talk/talk-1.fbx", - "https://lalaland.chat/animations/talk/talk-2.fbx", - "https://lalaland.chat/animations/talk/talk-3.fbx", - ], - }, - }, -]; - -interface VrmCompanionProps { - virtualText: string; - voiceUrl: string; - audioRef?: MutableRefObject; - onSpeakStart?: () => void; - onSpeakEnd?: () => void; +interface VrmAvatarProps { + meshRef?: MutableRefObject; + physicsRef?: MutableRefObject; + vrmUrl: string; + animations: Record< + "greet" | "idle" | "talk" | "bored" | "walk" | "happy" | "angry" | "sad", + string[] + >; + scale: number[]; + rotation?: number[]; + position?: number[]; + physics?: boolean; + isStaticPosition?: boolean; + gltfLoaded?: (gltf: GLTF) => void; } -const VrmCompanion = ({ - virtualText, - voiceUrl, - audioRef, - onSpeakStart, - onSpeakEnd, -}: VrmCompanionProps) => { - const [gltf, setGltf] = useState(null); - const [animationMixer, setAnimationMixer] = useState(null); - const [currentClip, setCurrentClip] = useState(null); - - const loader = useMemo(() => { - return new GLTFLoader().register( - (parser) => - new VRMLoaderPlugin(parser, { - autoUpdateHumanBones: true, - }) +const VrmCompanion = forwardRef( + ( + { + meshRef, + physicsRef, + vrmUrl, + animations, + scale, + rotation, + position, + physics, + isStaticPosition, + gltfLoaded, + }: VrmAvatarProps, + ref + ) => { + const [gltf, setGltf] = useState(null); + const [animationMixer, setAnimationMixer] = useState( + null ); - }, []); - - const randomVrm = useMemo( - () => vrms[Math.floor(Math.random() * vrms.length)], - [] - ); - - const gltfRef = useRef(); - const vrmRef = useRef(); - const textRef = useRef(null); - - useFrame(({ clock }, delta) => { - const s = Math.sin(Math.PI * clock.elapsedTime); - - const expressionManager = vrmRef.current?.expressionManager; - - if (expressionManager && animationMixer) { - // expressionManager.setValue("aa", 1.0); - vrmRef.current.update(clock.getDelta()); - } - - if (animationMixer?.update) { - animationMixer.update(delta); - } - if (vrmRef?.current?.update) { - vrmRef.current.update(delta); - } - - if (textRef.current) { - (textRef.current as any).position.y = s * 0.01; - } - }); - - const playAnimation = useCallback( - async ( - animationFile: string, - vrm: VRM, - currentMixer?: AnimationMixer, - oldClip?: AnimationClip - ) => { - const clip = await loadMixamoAnimation(animationFile, vrm); - - if (oldClip || currentClip) { - (currentMixer || animationMixer) - .clipAction(oldClip || currentClip) - .fadeOut(2); + const [prevVrmUrl, setPrevVrmUrl] = useState(null); + const [currentText, setCurrentText] = useState(""); + + const [targetPosition, setTargetPosition] = useState(position); + const [targetLookAt, setTargetLookAt] = useState(null); + const [animationCache, setAnimationCache] = useState< + Record + >({}); + const [audioContext, setAudioContext] = useState(null); + const [analyser, setAnalyser] = useState(null); + const [audio, setAudio] = useState(null); + + const loader = useMemo(() => { + return new GLTFLoader().register( + (parser: GLTFParser) => + new VRMLoaderPlugin(parser, { + autoUpdateHumanBones: true, + }) + ); + }, []); + + const rigidBodyRef = useRef(null); + const gltfRef = useRef(); + const vrmRef = useRef(); + const virtualTextRef = useRef(null); + + // bind refs to props for external access + useEffect(() => { + if (meshRef) { + meshRef.current = gltfRef.current; } - (currentMixer || animationMixer).clipAction(clip).fadeIn(2).play(); - setCurrentClip(clip); - - return clip; - }, - [animationMixer, currentClip] - ); - - const getRandomAnimation = useCallback( - (type: string) => { - const randomAnim = (randomVrm as any).animations?.[type]?.[ - Math.floor( - Math.random() * (randomVrm as any).animations?.[type]?.length - ) - ]; - - return randomAnim; - }, - [randomVrm.animations] - ); - - // load vrm and play greet animation - useEffect(() => { - if (!gltf) { - loader.loadAsync(randomVrm.model).then(async (gltf) => { - const vrm = gltf.userData.vrm as VRM; - VRMUtils.removeUnnecessaryJoints(vrm.scene); - VRMUtils.removeUnnecessaryVertices(vrm.scene); - - vrm.scene.traverse((obj) => { - obj.frustumCulled = false; - }); - - setGltf(gltf); - vrmRef.current = vrm; + if (physicsRef) { + physicsRef.current = rigidBodyRef.current; + } + }, [meshRef, physicsRef]); - const currentMixer = new AnimationMixer(vrm.scene); - currentMixer.timeScale = 1.0; - setAnimationMixer(currentMixer); + useFrame(({ camera }, delta) => { + if (animationMixer?.update) { + animationMixer.update(delta); + } + if (vrmRef?.current?.update) { + vrmRef.current.update(delta); + } - // wave greet - const randomGreet = getRandomAnimation("greet"); - const greetClip = await playAnimation(randomGreet, vrm, currentMixer); + if (virtualTextRef.current && gltfRef.current) { + const avatarPosition = new Vector3().setFromMatrixPosition( + gltfRef.current.matrixWorld + ); + virtualTextRef.current.position.copy(avatarPosition); + virtualTextRef.current.position.y += 1.5; + virtualTextRef.current.lookAt(camera.position); + } - // happy expression - const happyName = vrm.expressionManager.getExpressionTrackName("happy"); - const happyTrack = new NumberKeyframeTrack( - happyName, - [0.0, 0.4, 0.8, 1.2, 1.6, 2.0, 2.4, 2.8, 3.2, 3.6, 4.0], // times - [0.0, 0.2, 0.4, 0.6, 0.7, 0.7, 0.6, 0.6, 0.4, 0.2, 0.0] // values + if (gltfRef?.current?.matrixWorld && !isStaticPosition) { + const currentPosition = new Vector3().setFromMatrixPosition( + gltfRef.current.matrixWorld ); - const happyClip = new AnimationClip( - happyName, - 6.8, // duration - [happyTrack] + + const distance = currentPosition.distanceTo( + new Vector3(...targetPosition) ); - const happyAction = currentMixer.clipAction(happyClip); - happyAction.setLoop(LoopOnce, 1).play(); + + if (gltfRef.current && distance > 0.1) { + gltfRef.current.position.lerp(new Vector3(...targetPosition), 0.01); + } + } + if (gltfRef?.current?.lookAt && targetLookAt && !isStaticPosition) { + gltfRef.current.lookAt(new Vector3(...targetLookAt)); + gltfRef.current.rotateY(Math.PI); + } + }); + + const getRandomAnimation = useCallback( + (type: string) => { + const randomAnim = (animations as any)?.[type]?.[ + Math.floor(Math.random() * (animations as any)?.[type]?.length) + ]; + + return randomAnim; + }, + [animations] + ); + + const playAnimation = useCallback( + async (type: string) => { + animationCache[type][0].reset().setLoop(LoopOnce, 1).play(); + }, + [animationCache] + ); + + const moveMouth = useCallback( + async (audioUrl: string) => { + try { + const audioResp = await fetch(audioUrl); + const audioBuffer = await audioResp.arrayBuffer(); + const source = audioContext?.createBufferSource(); + const audio = await audioContext?.decodeAudioData(audioBuffer); + source.buffer = audio; + source?.connect(analyser); + source.start(0); + + const dataArray = new Uint8Array(analyser.frequencyBinCount); + + const updateMouth = () => { + requestAnimationFrame(updateMouth); + + analyser.getByteFrequencyData(dataArray); + + const volume = dataArray.reduce((a, b) => a + b) / dataArray.length; + const normalizationFactor = 50; + const normalizedVolume = Math.min(1, volume / normalizationFactor); + + // Set the weight of the 'Aa' blend shape based on the volume + vrmRef.current.expressionManager.setValue("aa", normalizedVolume); + vrmRef.current.expressionManager.update(); + }; + + updateMouth(); + } catch (error) { + console.error(error); + } + }, + [audioContext, analyser] + ); + + const setupAudioAnalyser = useCallback(async () => { + const audioContext = new (window.AudioContext || + (window as any).webkitAudioContext)(); + setAudioContext(audioContext); + + const analyser = audioContext?.createAnalyser(); + setAnalyser(analyser); + }, []); + + const setupAudioPlayer = useCallback(async () => { + const audio = new Audio(); + setAudio(audio); + }, []); + + const setupAnimations = useCallback(async () => { + return new Promise(async (resolve) => { + const mixer = new AnimationMixer(vrmRef.current.scene); + mixer.timeScale = 1.0; + setAnimationMixer(mixer); + + // load walk animation + const randomWalk = getRandomAnimation("walk"); + const walkClip = await loadMixamoAnimation(randomWalk, vrmRef.current); + const walkAction = mixer.clipAction(walkClip); + + // // load idle animation + const randomIdle = getRandomAnimation("idle"); + const idleClip = await loadMixamoAnimation(randomIdle, vrmRef.current); + const idleAction = mixer.clipAction(idleClip); + + setAnimationCache((prev) => ({ + ...prev, + walk: [...(prev?.walk || []), walkAction], + idle: [...(prev?.idle || []), idleAction], + })); + + idleAction.play(); // blink loop - const trackName = vrm.expressionManager.getExpressionTrackName("blink"); - const track = new NumberKeyframeTrack( - trackName, + const blinkTrack = + vrmRef.current.expressionManager.getExpressionTrackName("blink"); + const blinkKeys = new NumberKeyframeTrack( + blinkTrack as string, [0.0, 0.2, 0.4, 6.0], // times [0.0, 1.0, 0.0, 0.0] // values ); - const clip = new AnimationClip( - trackName, + const blinkClip = new AnimationClip( + blinkTrack as string, 6.8, // duration - [track] + [blinkKeys] ); - const action = currentMixer.clipAction(clip); + const action = mixer.clipAction(blinkClip); action.play(); - - // greet cleanup to idle - setTimeout(async () => { - const randomIdle = getRandomAnimation("idle"); - await playAnimation(randomIdle, vrm, currentMixer, greetClip); - }, greetClip.duration * 1000); + resolve(mixer); }); - } - }, [getRandomAnimation, gltf, loader, playAnimation, randomVrm.model]); - - useEffect(() => { - const main = async () => { - if (voiceUrl) { - onSpeakStart?.(); - audioRef.current.src = voiceUrl; - audioRef.current.play(); - - const randomTalk = getRandomAnimation("talk"); - const talkClip = await playAnimation(randomTalk, vrmRef.current); - - // lips movement - const trackName = - vrmRef.current.expressionManager.getExpressionTrackName("aa"); - const track = new NumberKeyframeTrack( - trackName, - [0.0, 0.5, 1.5, 2.0, 2.5], // times - [0.0, 1.0, 0.0, 1.0, 0.0] // values - ); - const clip = new AnimationClip( - trackName, - 2.5, // duration - [track] - ); - const lipsAction = animationMixer.clipAction(clip); - lipsAction.play(); - - audioRef.current.onended = async () => { - onSpeakEnd?.(); - animationMixer.clipAction(talkClip).fadeOut(2); - lipsAction.fadeOut(1); - const randomIdle = getRandomAnimation("idle"); - await playAnimation(randomIdle, vrmRef.current); - }; + }, [getRandomAnimation]); + + // load vrm and play greet animation + useEffect(() => { + if ((!gltf && vrmUrl) || prevVrmUrl !== vrmUrl) { + loader.loadAsync(vrmUrl).then(async (gltf: GLTF) => { + setPrevVrmUrl(vrmUrl); + const vrm = gltf.userData.vrm as VRM; + VRMUtils.removeUnnecessaryJoints(vrm.scene); + VRMUtils.removeUnnecessaryVertices(vrm.scene); + + vrm.scene.traverse((obj) => { + obj.frustumCulled = false; + }); + + const vrmScale = scale[0]; + + if (scale[0]) { + vrm.scene.scale.setScalar(scale[0]); + + // scale joints + for (const joint of vrm.springBoneManager.joints) { + joint.settings.stiffness *= vrmScale; + joint.settings.hitRadius *= vrmScale; + } + + // scale colliders + for (const collider of vrm.springBoneManager.colliders) { + const shape = collider.shape; + if (shape instanceof VRMSpringBoneColliderShapeCapsule) { + shape.radius *= vrmScale; + shape.tail.multiplyScalar(vrmScale); + } else if (shape instanceof VRMSpringBoneColliderShapeSphere) { + shape.radius *= vrmScale; + } + } + } + + setGltf(gltf); + + vrmRef.current = vrm; + + gltfLoaded?.(gltf); + + await setupAnimations(); + await setupAudioAnalyser(); + await setupAudioPlayer(); + }); } - }; - main(); - }, [voiceUrl]); - - return ( - <> - {gltf?.scene && ( - - - - {virtualText} - - - - - )} - - ); -}; - -const Scene = ({ - virtualText, - voiceUrl, - onSpeakStart, - onSpeakEnd, -}: VrmCompanionProps) => { - const audioRef = useRef(null); - - return ( - <> - - - - - - -