diff --git a/Social_AI_XR/Assets/FrameSynthesis.meta b/Social_AI_XR/Assets/FrameSynthesis.meta new file mode 100644 index 0000000..1eb9e90 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis.meta @@ -0,0 +1,5 @@ +fileFormatVersion: 2 +guid: 4d5000a8a89e52c4c8d499939663bf53 +folderAsset: yes +DefaultImporter: + userData: diff --git a/Social_AI_XR/Assets/FrameSynthesis/MyMath.cs b/Social_AI_XR/Assets/FrameSynthesis/MyMath.cs new file mode 100644 index 0000000..2fd14b0 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/MyMath.cs @@ -0,0 +1,21 @@ +using UnityEngine; + +namespace FrameSynthesis +{ + public class MyMath + { + public static float LinearMap(float value, float s0, float s1, float d0, float d1) + { + return d0 + (value - s0) * (d1 - d0) / (s1 - s0); + } + + public static float WrapDegree(float degree) + { + if (degree > 180f) + { + return degree - 360f; + } + return degree; + } + } +} \ No newline at end of file diff --git a/Social_AI_XR/Assets/FrameSynthesis/MyMath.cs.meta b/Social_AI_XR/Assets/FrameSynthesis/MyMath.cs.meta new file mode 100644 index 0000000..22847ea --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/MyMath.cs.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 05b33d9d734fa314ba20df196d84992c +MonoImporter: + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer.meta new file mode 100644 index 0000000..8f7a16c --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: b345acec6a0958f4db8b092d24d5dfbf +folderAsset: yes +DefaultImporter: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example.meta new file mode 100644 index 0000000..6f5004c --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: 2e72abbcf6561b84cb6d0a3e1515b2e6 +folderAsset: yes +timeCreated: 1507562349 +licenseType: Pro +DefaultImporter: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph.meta new file mode 100644 index 0000000..fb8935b --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 4e4afd144a0cfab488982b8d964ae759 +folderAsset: yes +DefaultImporter: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.mat b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.mat new file mode 100644 index 0000000..ebb11ba Binary files /dev/null and b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.mat differ diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.mat.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.mat.meta new file mode 100644 index 0000000..47b98c3 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.mat.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: cc341bb2ea57d724887459ae6c8eeac2 +NativeFormatImporter: + mainObjectFileID: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.prefab b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.prefab new file mode 100644 index 0000000..7f343a7 Binary files /dev/null and b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.prefab differ diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.prefab.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.prefab.meta new file mode 100644 index 0000000..cad8e74 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.prefab.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: a4bb33488e108694bbf176bd5cbc00e7 +NativeFormatImporter: + mainObjectFileID: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.renderTexture b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.renderTexture new file mode 100644 index 0000000..fd4001b Binary files /dev/null and b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.renderTexture differ diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.renderTexture.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.renderTexture.meta new file mode 100644 index 0000000..cea0f76 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraph.renderTexture.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 3d06d1a28a25997448ddf91405f51fa0 +NativeFormatImporter: + mainObjectFileID: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraphRenderer.cs b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraphRenderer.cs new file mode 100644 index 0000000..b30db9e --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraphRenderer.cs @@ -0,0 +1,78 @@ +using UnityEngine; + +namespace FrameSynthesis.VR +{ + public class DebugGraphRenderer : MonoBehaviour + { + [SerializeField] + Material material; + + void OnPostRender() + { + material.SetPass(0); + + GL.LoadOrtho(); + GL.Begin(GL.LINES); + + GL.Color(new Color(1f, 1f, 1f, 0.5f)); + GL.Vertex3(0f, 0.5f, 0f); + GL.Vertex3(1f, 0.5f, 0f); + GL.Vertex3(0.5f, 0f, 0f); + GL.Vertex3(0.5f, 1f, 0f); + + var poseSamples = VRGestureRecognizer.Current.PoseSamples; + + GL.Color(Color.red); + + var prevGraphPosition = Vector2.zero; + var i = 0; + foreach (var poseSample in poseSamples) + { + var graphPosition = GetGraphPositionFromPoseSamplePitch(poseSample); + if (i > 0) + { + GL.Vertex3(prevGraphPosition.x, prevGraphPosition.y, 0f); + GL.Vertex3(graphPosition.x, graphPosition.y, 0f); + } + prevGraphPosition = graphPosition; + i++; + } + + GL.Color(Color.green); + + i = 0; + foreach (var poseSample in poseSamples) + { + var graphPosition = GetGraphPositionFromPoseSampleYaw(poseSample); + if (i > 0) + { + GL.Vertex3(prevGraphPosition.x, prevGraphPosition.y, 0f); + GL.Vertex3(graphPosition.x, graphPosition.y, 0f); + } + prevGraphPosition = graphPosition; + i++; + } + + GL.End(); + } + + Vector2 GetGraphPositionFromPoseSamplePitch(PoseSample poseSample) + { + float x = Time.time - poseSample.timestamp; + float y = ProjectDegreeTo01(poseSample.orientation.eulerAngles.x); + return new Vector2(x, y); + } + + Vector2 GetGraphPositionFromPoseSampleYaw(PoseSample poseSample) + { + float x = ProjectDegreeTo01(poseSample.orientation.eulerAngles.y); + float y = Time.time - poseSample.timestamp; + return new Vector2(x, y); + } + + float ProjectDegreeTo01(float degree) + { + return MyMath.LinearMap(MyMath.WrapDegree(degree), -180f, 180f, 0f, 1f); + } + } +} diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraphRenderer.cs.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraphRenderer.cs.meta new file mode 100644 index 0000000..f728352 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/DebugGraphRenderer.cs.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: 1e42bfd12228bd84cb74deefb7011851 +MonoImporter: + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.mat b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.mat new file mode 100644 index 0000000..238a294 Binary files /dev/null and b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.mat differ diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.mat.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.mat.meta new file mode 100644 index 0000000..4c1dc2d --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.mat.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 446a579c8c6eaa94595ae0ec0952c47b +NativeFormatImporter: + mainObjectFileID: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.shader b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.shader new file mode 100644 index 0000000..19edd30 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.shader @@ -0,0 +1,16 @@ +Shader "Frame Synthesis/GL Line" +{ + SubShader + { + Pass + { + Blend SrcAlpha OneMinusSrcAlpha + ZWrite Off Cull Off Fog { Mode Off } + BindChannels + { + Bind "vertex", vertex + Bind "color", color + } + } + } +} diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.shader.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.shader.meta new file mode 100644 index 0000000..df12f1e --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/DebugGraph/GLLine.shader.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: acbaa3fba10266f42a6644998e12f88f +ShaderImporter: + defaultTextures: [] + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Example.unity b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Example.unity new file mode 100644 index 0000000..fef15b4 Binary files /dev/null and b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Example.unity differ diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Example.unity.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Example.unity.meta new file mode 100644 index 0000000..cd2e852 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Example.unity.meta @@ -0,0 +1,4 @@ +fileFormatVersion: 2 +guid: f30e119b54bc6ad468dda3066903bf16 +DefaultImporter: + userData: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Prefabs.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Prefabs.meta new file mode 100644 index 0000000..b2d88ee --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Prefabs.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: 2da76a64c47567a459c10ff82c3fcdea +folderAsset: yes +timeCreated: 1507562147 +licenseType: Pro +DefaultImporter: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Prefabs/YesNoMessage.prefab b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Prefabs/YesNoMessage.prefab new file mode 100644 index 0000000..3fe10a0 Binary files /dev/null and b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Prefabs/YesNoMessage.prefab differ diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Prefabs/YesNoMessage.prefab.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Prefabs/YesNoMessage.prefab.meta new file mode 100644 index 0000000..a0f72f6 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Prefabs/YesNoMessage.prefab.meta @@ -0,0 +1,4 @@ +fileFormatVersion: 2 +guid: 2a798dfd08ee0f54f9828c3677181a19 +NativeFormatImporter: + userData: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts.meta new file mode 100644 index 0000000..6f226cf --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: f8313bd8259e4864fba60e743f2fcc79 +folderAsset: yes +timeCreated: 1507562372 +licenseType: Pro +DefaultImporter: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessage.cs b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessage.cs new file mode 100644 index 0000000..73e32b5 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessage.cs @@ -0,0 +1,31 @@ +using UnityEngine; + +namespace FrameSynthesis.VR.Example +{ + [RequireComponent(typeof(TextMesh))] + public class YesNoMessage : MonoBehaviour + { + const string YesText = "Yes!"; + const string NoText = "No!"; + + [SerializeField] + float lifetime = 1f; + [SerializeField] + Vector3 velocity; + + void Start() + { + Destroy(gameObject, lifetime); + } + + public void Initialize(bool yesNo) + { + GetComponent().text = yesNo ? YesText : NoText; + } + + void Update() + { + transform.Translate(velocity * Time.deltaTime); + } + } +} diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessage.cs.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessage.cs.meta new file mode 100644 index 0000000..5719941 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessage.cs.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: ed8e5ac8a24c7404293a3416019b424c +MonoImporter: + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessageGenerator.cs b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessageGenerator.cs new file mode 100644 index 0000000..443cd1c --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessageGenerator.cs @@ -0,0 +1,36 @@ +using UnityEngine; + +namespace FrameSynthesis.VR.Example +{ + public class YesNoMessageGenerator : MonoBehaviour + { + [SerializeField] + GameObject yesNoMessagePrefab; + + void Start() + { + VRGestureRecognizer.Current.NodHandler += OnNod; + VRGestureRecognizer.Current.HeadshakeHandler += OnHeadshake; + } + + void OnNod() + { + InstantiateYesNoMessage(true); + } + + void OnHeadshake() + { + InstantiateYesNoMessage(false); + } + + void InstantiateYesNoMessage(bool yesNo) + { + var go = Instantiate(yesNoMessagePrefab); + go.GetComponent().Initialize(yesNo); + + go.transform.SetParent(Camera.main.transform); + go.transform.localPosition = new Vector3(0f, 0f, 10f); + go.transform.localRotation = Quaternion.identity; + } + } +} diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessageGenerator.cs.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessageGenerator.cs.meta new file mode 100644 index 0000000..2f56e31 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Example/Scripts/YesNoMessageGenerator.cs.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 278baf2913811094892291487316f023 +MonoImporter: + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Recenterer.cs b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Recenterer.cs new file mode 100644 index 0000000..c368b05 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Recenterer.cs @@ -0,0 +1,21 @@ +using UnityEngine; +using UnityEngine.XR; + +namespace FrameSynthesis.VR +{ + public class Recenterer : MonoBehaviour + { + void Start() + { + XRDevice.SetTrackingSpaceType(TrackingSpaceType.Stationary); + } + + void Update() + { + if (Input.GetKeyDown(KeyCode.R)) + { + InputTracking.Recenter(); + } + } + } +} diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Recenterer.cs.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Recenterer.cs.meta new file mode 100644 index 0000000..d2d1d6f --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/Recenterer.cs.meta @@ -0,0 +1,12 @@ +fileFormatVersion: 2 +guid: 0fd0fb61c246f114eae3cb1d0477a38f +timeCreated: 1507555191 +licenseType: Pro +MonoImporter: + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/VRGestureRecognizer.cs b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/VRGestureRecognizer.cs new file mode 100644 index 0000000..de31852 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/VRGestureRecognizer.cs @@ -0,0 +1,118 @@ +using UnityEngine; +using UnityEngine.XR; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace FrameSynthesis.VR +{ + public struct PoseSample + { + public float timestamp; + public Quaternion orientation; + public Vector3 eulerAngles; + + public PoseSample(float timestamp, Quaternion orientation) + { + this.timestamp = timestamp; + this.orientation = orientation; + + eulerAngles = orientation.eulerAngles; + eulerAngles.x = MyMath.WrapDegree(eulerAngles.x); + eulerAngles.y = MyMath.WrapDegree(eulerAngles.y); + } + } + + public class VRGestureRecognizer : MonoBehaviour + { + public static VRGestureRecognizer Current { get; private set; } + + [SerializeField] + float recognitionInterval = 0.5f; + + public event Action NodHandler; + public event Action HeadshakeHandler; + + public Queue PoseSamples { get; } = new Queue(); + + float prevGestureTime; + + void Awake() + { + Current = this; + } + + void Update() + { + var orientation = InputTracking.GetLocalRotation(XRNode.Head); + + // Record orientation + PoseSamples.Enqueue(new PoseSample(Time.time, orientation)); + if (PoseSamples.Count >= 120) + { + PoseSamples.Dequeue(); + } + + // Recognize gestures + RecognizeNod(); + RecognizeHeadshake(); + } + + IEnumerable PoseSamplesWithin(float startTime, float endTime) + { + return PoseSamples.Where(sample => + sample.timestamp < Time.time - startTime && + sample.timestamp >= Time.time - endTime); + } + + void RecognizeNod() + { + try + { + var averagePitch = PoseSamplesWithin(0.2f, 0.4f).Average(sample => sample.eulerAngles.x); + var maxPitch = PoseSamplesWithin(0.01f, 0.2f).Max(sample => sample.eulerAngles.x); + var pitch = PoseSamples.First().eulerAngles.x; + + if (maxPitch - averagePitch > 10f && + Mathf.Abs(pitch - averagePitch) < 5f) + { + if (prevGestureTime < Time.time - recognitionInterval) + { + prevGestureTime = Time.time; + NodHandler?.Invoke(); + } + } + } + catch (InvalidOperationException) + { + // PoseSamplesWithin contains no entry + } + } + + void RecognizeHeadshake() + { + try + { + var averageYaw = PoseSamplesWithin(0.2f, 0.4f).Average(sample => sample.eulerAngles.y); + var maxYaw = PoseSamplesWithin(0.01f, 0.2f).Max(sample => sample.eulerAngles.y); + var minYaw = PoseSamplesWithin(0.01f, 0.2f).Min(sample => sample.eulerAngles.y); + var yaw = PoseSamples.First().eulerAngles.y; + + if ((maxYaw - averageYaw > 10f || averageYaw - minYaw > 10f) && + Mathf.Abs(yaw - averageYaw) < 5f) + { + if (prevGestureTime < Time.time - recognitionInterval) + { + prevGestureTime = Time.time; + HeadshakeHandler?.Invoke(); + } + } + } + catch (InvalidOperationException) + { + // PoseSamplesWithin contains no entry + } + } + } +} + diff --git a/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/VRGestureRecognizer.cs.meta b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/VRGestureRecognizer.cs.meta new file mode 100644 index 0000000..0939172 --- /dev/null +++ b/Social_AI_XR/Assets/FrameSynthesis/VRGestureRecognizer/VRGestureRecognizer.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: f90909e5f08798444aa89e4fcc2d9dea +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/ScriptExample.meta b/Social_AI_XR/Assets/ScriptExample.meta new file mode 100644 index 0000000..7d91471 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: c6b532bb92ca88846aec1a66b60a10e1 +folderAsset: yes +timeCreated: 1507562069 +licenseType: Pro +DefaultImporter: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/ScriptExample/Audio.meta b/Social_AI_XR/Assets/ScriptExample/Audio.meta new file mode 100644 index 0000000..a27110a --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Audio.meta @@ -0,0 +1,5 @@ +fileFormatVersion: 2 +guid: 628ed10993205bf44afcbaf6d6039bf5 +folderAsset: yes +DefaultImporter: + userData: diff --git a/Social_AI_XR/Assets/ScriptExample/Audio/gesture.wav b/Social_AI_XR/Assets/ScriptExample/Audio/gesture.wav new file mode 100644 index 0000000..b75b92e Binary files /dev/null and b/Social_AI_XR/Assets/ScriptExample/Audio/gesture.wav differ diff --git a/Social_AI_XR/Assets/ScriptExample/Audio/gesture.wav.meta b/Social_AI_XR/Assets/ScriptExample/Audio/gesture.wav.meta new file mode 100644 index 0000000..9838be6 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Audio/gesture.wav.meta @@ -0,0 +1,12 @@ +fileFormatVersion: 2 +guid: ec4611231a033f7488b933bbc0b771c7 +AudioImporter: + serializedVersion: 4 + format: 0 + quality: .5 + stream: 0 + 3D: 0 + forceToMono: 0 + useHardware: 0 + loopable: 0 + userData: diff --git a/Social_AI_XR/Assets/ScriptExample/Audio/po.wav b/Social_AI_XR/Assets/ScriptExample/Audio/po.wav new file mode 100644 index 0000000..36fb001 Binary files /dev/null and b/Social_AI_XR/Assets/ScriptExample/Audio/po.wav differ diff --git a/Social_AI_XR/Assets/ScriptExample/Audio/po.wav.meta b/Social_AI_XR/Assets/ScriptExample/Audio/po.wav.meta new file mode 100644 index 0000000..a31ad4c --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Audio/po.wav.meta @@ -0,0 +1,12 @@ +fileFormatVersion: 2 +guid: 85e8f4ead74bdef4ba95f952598e8029 +AudioImporter: + serializedVersion: 4 + format: 0 + quality: .5 + stream: 0 + 3D: 0 + forceToMono: 0 + useHardware: 0 + loopable: 0 + userData: diff --git a/Social_AI_XR/Assets/ScriptExample/Materials.meta b/Social_AI_XR/Assets/ScriptExample/Materials.meta new file mode 100644 index 0000000..4f23524 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Materials.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: 499bb2d8dc25237499d1b1e2b18b6ea4 +folderAsset: yes +timeCreated: 1507562110 +licenseType: Pro +DefaultImporter: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/ScriptExample/Materials/Window.mat b/Social_AI_XR/Assets/ScriptExample/Materials/Window.mat new file mode 100644 index 0000000..4bbd084 Binary files /dev/null and b/Social_AI_XR/Assets/ScriptExample/Materials/Window.mat differ diff --git a/Social_AI_XR/Assets/ScriptExample/Materials/Window.mat.meta b/Social_AI_XR/Assets/ScriptExample/Materials/Window.mat.meta new file mode 100644 index 0000000..87de50a --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Materials/Window.mat.meta @@ -0,0 +1,4 @@ +fileFormatVersion: 2 +guid: a838c0812fc77414b80a3f0e2f2b4ebb +NativeFormatImporter: + userData: diff --git a/Social_AI_XR/Assets/ScriptExample/Resources.meta b/Social_AI_XR/Assets/ScriptExample/Resources.meta new file mode 100644 index 0000000..021e2b5 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Resources.meta @@ -0,0 +1,5 @@ +fileFormatVersion: 2 +guid: 4e2aee29186805346a2b1bd7cb093cb6 +folderAsset: yes +DefaultImporter: + userData: diff --git a/Social_AI_XR/Assets/ScriptExample/Resources/script.txt b/Social_AI_XR/Assets/ScriptExample/Resources/script.txt new file mode 100644 index 0000000..9e3ed92 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Resources/script.txt @@ -0,0 +1,21 @@ +delay 500 + +*start +わたしを お城まで 連れて +帰ってくださいますのね？ +gesture *yes, *no + +*yes +delay 300 +うれしゅうございます。 +ぽっ…。 +delay 1500 +goto *start + +*no +delay 300 +そんな ひどい…。 +delay 1500 +goto *start + + diff --git a/Social_AI_XR/Assets/ScriptExample/Resources/script.txt.meta b/Social_AI_XR/Assets/ScriptExample/Resources/script.txt.meta new file mode 100644 index 0000000..9d65749 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Resources/script.txt.meta @@ -0,0 +1,4 @@ +fileFormatVersion: 2 +guid: 2498aa33a8c99dd4ab8280dd0615337f +TextScriptImporter: + userData: diff --git a/Social_AI_XR/Assets/ScriptExample/ScriptExample.unity b/Social_AI_XR/Assets/ScriptExample/ScriptExample.unity new file mode 100644 index 0000000..7dbf960 Binary files /dev/null and b/Social_AI_XR/Assets/ScriptExample/ScriptExample.unity differ diff --git a/Social_AI_XR/Assets/ScriptExample/ScriptExample.unity.meta b/Social_AI_XR/Assets/ScriptExample/ScriptExample.unity.meta new file mode 100644 index 0000000..ac80bf9 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/ScriptExample.unity.meta @@ -0,0 +1,4 @@ +fileFormatVersion: 2 +guid: c7efa4992f143f54091bb2f14d72a56b +DefaultImporter: + userData: diff --git a/Social_AI_XR/Assets/ScriptExample/Scripts.meta b/Social_AI_XR/Assets/ScriptExample/Scripts.meta new file mode 100644 index 0000000..30c5a13 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Scripts.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: b82320d0d1c466e458406508620a85d0 +folderAsset: yes +timeCreated: 1507562093 +licenseType: Pro +DefaultImporter: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/ScriptExample/Scripts/Main.cs b/Social_AI_XR/Assets/ScriptExample/Scripts/Main.cs new file mode 100644 index 0000000..7cbdf99 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Scripts/Main.cs @@ -0,0 +1,37 @@ +using UnityEngine; +using FrameSynthesis.VR; + +namespace ScriptExample +{ + public class Main : MonoBehaviour + { + [SerializeField] + ScriptEngine scriptEngine; + [SerializeField] + AudioSource gestureSound; + + void Start() + { + VRGestureRecognizer.Current.NodHandler += OnNod; + VRGestureRecognizer.Current.HeadshakeHandler += OnHeadshake; + } + + void OnNod() + { + if (scriptEngine.IsWaitingYesNo) + { + scriptEngine.AnswerYes(); + gestureSound.Play(); + } + } + + void OnHeadshake() + { + if (scriptEngine.IsWaitingYesNo) + { + scriptEngine.AnswerNo(); + gestureSound.Play(); + } + } + } +} diff --git a/Social_AI_XR/Assets/ScriptExample/Scripts/Main.cs.meta b/Social_AI_XR/Assets/ScriptExample/Scripts/Main.cs.meta new file mode 100644 index 0000000..0921efd --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Scripts/Main.cs.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: d5f420d93a3a29e4586662ef429ce895 +MonoImporter: + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: diff --git a/Social_AI_XR/Assets/ScriptExample/Scripts/MessageWindow.cs b/Social_AI_XR/Assets/ScriptExample/Scripts/MessageWindow.cs new file mode 100644 index 0000000..0298c04 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Scripts/MessageWindow.cs @@ -0,0 +1,63 @@ +using UnityEngine; + +namespace ScriptExample +{ + public class MessageWindow : MonoBehaviour + { + [SerializeField] + ScriptEngine scriptEngine; + + [SerializeField] + float interval = 0.05f; + + [SerializeField] + TextMesh textMesh; + [SerializeField] + AudioSource audioSource; + + bool running; + + string message; + float messageStartTime; + + int prevCursor; + + void Start() + { + scriptEngine.ShowMessageHandler += OnShowMessage; + } + + void Update() + { + if (!running) return; + + var cursor = (int)((Time.time - messageStartTime) / interval); + if (cursor >= message.Length) + { + running = false; + scriptEngine.NextCommand(); + return; + } + + textMesh.text = message.Substring(0, cursor + 1); + + if (prevCursor != cursor) + { + prevCursor = cursor; + var letter = message[cursor]; + if (letter != 32 && letter != 10) + { + audioSource.Play(); + } + } + } + + void OnShowMessage(string message) + { + this.message = message; + messageStartTime = Time.time; + running = true; + prevCursor = -1; + } + } +} diff --git a/Social_AI_XR/Assets/ScriptExample/Scripts/MessageWindow.cs.meta b/Social_AI_XR/Assets/ScriptExample/Scripts/MessageWindow.cs.meta new file mode 100644 index 0000000..4f70a49 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Scripts/MessageWindow.cs.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 6cad35a15e546e54d93a35eed9664799 +MonoImporter: + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: diff --git a/Social_AI_XR/Assets/ScriptExample/Scripts/ScriptEngine.cs b/Social_AI_XR/Assets/ScriptExample/Scripts/ScriptEngine.cs new file mode 100644 index 0000000..4fcc794 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Scripts/ScriptEngine.cs @@ -0,0 +1,161 @@ +using UnityEngine; +using System; +using System.Collections.Generic; +using System.Text.RegularExpressions; + +namespace ScriptExample +{ + public class ScriptEngine : MonoBehaviour + { + public bool IsWaitingYesNo => state == State.WaitingYesNo; + + public event Action ShowMessageHandler; + + int cursor; + string[] lines; + + readonly string[] commands = { + "gesture", + "delay", + "goto" + }; + + public enum State + { + Normal, + WaitingYesNo, + } + + State state; + + string yesLabel; + string noLabel; + + readonly Dictionary labels = new Dictionary(); + + void Start() + { + var textAsset = Resources.Load("script") as TextAsset; + lines = textAsset.text.Split('\n'); + + for (var i = 0; i < lines.Length; i++) + { + // Trim + var line = lines[i].Trim(); + + // Comment + if (line.StartsWith(";")) + { + line = ""; + } + + // Label + if (line.StartsWith("*")) + { + labels.Add(line.Substring(1), i); + line = ""; + } + lines[i] = line; + } + + state = State.Normal; + + NextCommand(); + } + + string GetCommand(string line) + { + foreach (var command in commands) + { + if (line.StartsWith(command)) + { + return command; + } + } + return ""; + } + + public void NextCommand() + { + var command = GetCommand(lines[cursor]); + switch (command) + { + case "gesture": + var r = new Regex(@"^gesture\s+\*(\w+)\s*,\s*\*(\w+)$"); + var m = r.Match(lines[cursor]); + + if (m.Groups.Count == 3) + { + yesLabel = m.Groups[1].ToString(); + noLabel = m.Groups[2].ToString(); + state = State.WaitingYesNo; + return; + } + throw new Exception("Syntax error"); + + case "delay": + r = new Regex(@"^delay\s+(\d+)$"); + m = r.Match(lines[cursor]); + + if (m.Groups.Count == 2) + { + cursor++; + var time = int.Parse(m.Groups[1].ToString()); + Invoke(nameof(NextCommand), time / 1000f); + return; + } + throw new Exception("Syntax error"); + + case "goto": + r = new Regex(@"^goto\s+\*(\w+)$"); + m = r.Match(lines[cursor]); + + if (m.Groups.Count == 2) + { + var label = m.Groups[1].ToString(); + GoTo(label); + return; + } + throw new Exception("Syntax error"); + } + + var message = ""; + + while (GetCommand(lines[cursor]) == "") + { + string line = lines[cursor]; + if (line != "") + { + message += line + "\n"; + } + cursor++; + } + + ShowMessageHandler?.Invoke(message); + } + + public void GoTo(string label) + { + cursor = labels[label]; + NextCommand(); + } + + public void AnswerYes() + { + if (state == State.WaitingYesNo) + { + state = State.Normal; + GoTo(yesLabel); + } + } + + public void AnswerNo() + { + if (state == State.WaitingYesNo) + { + state = State.Normal; + GoTo(noLabel); + } + } + } +} diff --git a/Social_AI_XR/Assets/ScriptExample/Scripts/ScriptEngine.cs.meta b/Social_AI_XR/Assets/ScriptExample/Scripts/ScriptEngine.cs.meta new file mode 100644 index 0000000..673ae34 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Scripts/ScriptEngine.cs.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: ddceca155a8ec394187ad9218fa477a8 +MonoImporter: + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: diff --git a/Social_AI_XR/Assets/ScriptExample/Textures.meta b/Social_AI_XR/Assets/ScriptExample/Textures.meta new file mode 100644 index 0000000..7c18262 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Textures.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: acb34574d9e767c4c99ae237d2ffa66b +folderAsset: yes +timeCreated: 1507562121 +licenseType: Pro +DefaultImporter: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Social_AI_XR/Assets/ScriptExample/Textures/Window.png b/Social_AI_XR/Assets/ScriptExample/Textures/Window.png new file mode 100644 index 0000000..a70f4a7 Binary files /dev/null and b/Social_AI_XR/Assets/ScriptExample/Textures/Window.png differ diff --git a/Social_AI_XR/Assets/ScriptExample/Textures/Window.png.meta b/Social_AI_XR/Assets/ScriptExample/Textures/Window.png.meta new file mode 100644 index 0000000..370ed14 --- /dev/null +++ b/Social_AI_XR/Assets/ScriptExample/Textures/Window.png.meta @@ -0,0 +1,45 @@ +fileFormatVersion: 2 +guid: 81530f30f7bba05429a1d73be2333835 +TextureImporter: + serializedVersion: 2 + mipmaps: + mipMapMode: 0 + enableMipMap: 0 + linearTexture: 1 + correctGamma: 0 + fadeOut: 0 + borderMipMap: 0 + mipMapFadeDistanceStart: 1 + mipMapFadeDistanceEnd: 3 + bumpmap: + convertToNormalMap: 0 + externalNormalMap: 0 + heightScale: .25 + normalMapFilter: 0 + isReadable: 0 + grayScaleToAlpha: 0 + generateCubemap: 0 + seamlessCubemap: 0 + textureFormat: -1 + maxTextureSize: 1024 + textureSettings: + filterMode: -1 + aniso: 1 + mipBias: -1 + wrapMode: 1 + nPOTScale: 0 + lightmap: 0 + compressionQuality: 50 + spriteMode: 0 + spriteExtrude: 1 + spriteMeshType: 1 + alignment: 0 + spritePivot: {x: .5, y: .5} + spritePixelsToUnits: 100 + alphaIsTransparency: 1 + textureType: 2 + buildTargetSettings: [] + spriteSheet: + sprites: [] + spritePackingTag: + userData: