Assignment for RMIT Mixed Reality in 2020
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

162 lines
5.5 KiB

  1. /************************************************************************************
  2. Filename : OVRLipSyncSequence.cs
  3. Content : LipSync frames container
  4. Created : May 17th, 2018
  5. Copyright : Copyright Facebook Technologies, LLC and its affiliates.
  6. All rights reserved.
  7. Licensed under the Oculus Audio SDK License Version 3.3 (the "License");
  8. you may not use the Oculus Audio SDK except in compliance with the License,
  9. which is provided at the time of installation or download, or which
  10. otherwise accompanies this software in either electronic or hard copy form.
  11. You may obtain a copy of the License at
  12. https://developer.oculus.com/licenses/audio-3.3/
  13. Unless required by applicable law or agreed to in writing, the Oculus Audio SDK
  14. distributed under the License is distributed on an "AS IS" BASIS,
  15. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16. See the License for the specific language governing permissions and
  17. limitations under the License.
  18. ************************************************************************************/
  19. using System;
  20. using System.Collections.Generic;
  21. using System.IO;
  22. using UnityEngine;
  23. // Sequence - holds ordered entries for playback
  24. [System.Serializable]
  25. public class OVRLipSyncSequence : ScriptableObject
  26. {
  27. public List<OVRLipSync.Frame> entries = new List<OVRLipSync.Frame>();
  28. public float length; // in seconds
  29. public OVRLipSync.Frame GetFrameAtTime(float time)
  30. {
  31. OVRLipSync.Frame frame = null;
  32. if (time < length && entries.Count > 0)
  33. {
  34. float percentComplete = time / length;
  35. frame = entries[(int)(entries.Count * percentComplete)];
  36. }
  37. return frame;
  38. }
  39. #if UNITY_EDITOR
  40. private static readonly int sSampleSize = 1024;
  41. public static OVRLipSyncSequence CreateSequenceFromAudioClip(
  42. AudioClip clip, bool useOfflineModel = false)
  43. {
  44. OVRLipSyncSequence sequence = null;
  45. if (clip.channels > 2)
  46. {
  47. Debug.LogError(clip.name +
  48. ": Cannot process phonemes from an audio clip with " +
  49. "more than 2 channels");
  50. return null;
  51. }
  52. if (clip.loadType != AudioClipLoadType.DecompressOnLoad)
  53. {
  54. Debug.LogError(clip.name +
  55. ": Cannot process phonemes from an audio clip unless " +
  56. "its load type is set to DecompressOnLoad.");
  57. return null;
  58. }
  59. if (OVRLipSync.Initialize(clip.frequency, sSampleSize) != OVRLipSync.Result.Success)
  60. {
  61. Debug.LogError("Could not create Lip Sync engine.");
  62. return null;
  63. }
  64. if (clip.loadState != AudioDataLoadState.Loaded)
  65. {
  66. Debug.LogError("Clip is not loaded!");
  67. return null;
  68. }
  69. uint context = 0;
  70. OVRLipSync.Result result = useOfflineModel
  71. ? OVRLipSync.CreateContextWithModelFile(
  72. ref context,
  73. OVRLipSync.ContextProviders.Enhanced,
  74. Path.Combine(Application.dataPath, "Oculus/LipSync/Assets/OfflineModel/ovrlipsync_offline_model.pb"))
  75. : OVRLipSync.CreateContext(ref context, OVRLipSync.ContextProviders.Enhanced);
  76. if (result != OVRLipSync.Result.Success)
  77. {
  78. Debug.LogError("Could not create Phoneme context. (" + result + ")");
  79. OVRLipSync.Shutdown();
  80. return null;
  81. }
  82. List<OVRLipSync.Frame> frames = new List<OVRLipSync.Frame>();
  83. float[] samples = new float[sSampleSize * clip.channels];
  84. OVRLipSync.Frame dummyFrame = new OVRLipSync.Frame();
  85. OVRLipSync.ProcessFrame(
  86. context,
  87. samples,
  88. dummyFrame,
  89. (clip.channels == 2) ? true : false
  90. );
  91. // frame delay in ms
  92. float frameDelayInMs = dummyFrame.frameDelay;
  93. int frameOffset = (int)(frameDelayInMs * clip.frequency / 1000);
  94. int totalSamples = clip.samples;
  95. for (int x = 0; x < totalSamples + frameOffset; x += sSampleSize)
  96. {
  97. int remainingSamples = totalSamples - x;
  98. if (remainingSamples >= sSampleSize) {
  99. clip.GetData(samples, x);
  100. } else if (remainingSamples > 0) {
  101. float[] samples_clip = new float[remainingSamples * clip.channels];
  102. clip.GetData(samples_clip, x);
  103. Array.Copy(samples_clip, samples, samples_clip.Length);
  104. Array.Clear(samples, samples_clip.Length, samples.Length - samples_clip.Length);
  105. } else {
  106. Array.Clear(samples, 0, samples.Length);
  107. }
  108. OVRLipSync.Frame frame = new OVRLipSync.Frame();
  109. if (clip.channels == 2)
  110. {
  111. // interleaved = stereo data, alternating floats
  112. OVRLipSync.ProcessFrame(context, samples, frame);
  113. }
  114. else
  115. {
  116. // mono
  117. OVRLipSync.ProcessFrame(context, samples, frame, false);
  118. }
  119. if (x < frameOffset)
  120. {
  121. continue;
  122. }
  123. frames.Add(frame);
  124. }
  125. Debug.Log(clip.name + " produced " + frames.Count +
  126. " viseme frames, playback rate is " + (frames.Count / clip.length) +
  127. " fps");
  128. OVRLipSync.DestroyContext(context);
  129. OVRLipSync.Shutdown();
  130. sequence = ScriptableObject.CreateInstance<OVRLipSyncSequence>();
  131. sequence.entries = frames;
  132. sequence.length = clip.length;
  133. return sequence;
  134. }
  135. #endif
  136. };