You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
441 lines
15 KiB
441 lines
15 KiB
/************************************************************************************
|
|
Filename : OVRLipSync.cs
|
|
Content : Interface to Oculus Lip Sync engine
|
|
Created : August 4th, 2015
|
|
Copyright : Copyright Facebook Technologies, LLC and its affiliates.
|
|
All rights reserved.
|
|
|
|
Licensed under the Oculus Audio SDK License Version 3.3 (the "License");
|
|
you may not use the Oculus Audio SDK except in compliance with the License,
|
|
which is provided at the time of installation or download, or which
|
|
otherwise accompanies this software in either electronic or hard copy form.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
https://developer.oculus.com/licenses/audio-3.3/
|
|
|
|
Unless required by applicable law or agreed to in writing, the Oculus Audio SDK
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
************************************************************************************/
|
|
using UnityEngine;
|
|
using System;
|
|
using System.Runtime.InteropServices;
|
|
|
|
//-------------------------------------------------------------------------------------
|
|
// ***** OVRLipSync
|
|
//
|
|
/// <summary>
|
|
/// OVRLipSync interfaces into the Oculus lip sync engine. This component should be added
|
|
/// into the scene once.
|
|
///
|
|
/// </summary>
|
|
public class OVRLipSync : MonoBehaviour
|
|
{
|
|
// Error codes that may return from Lip Sync engine
|
|
public enum Result
|
|
{
|
|
Success = 0,
|
|
Unknown = -2200, //< An unknown error has occurred
|
|
CannotCreateContext = -2201, //< Unable to create a context
|
|
InvalidParam = -2202, //< An invalid parameter, e.g. NULL pointer or out of range
|
|
BadSampleRate = -2203, //< An unsupported sample rate was declared
|
|
MissingDLL = -2204, //< The DLL or shared library could not be found
|
|
BadVersion = -2205, //< Mismatched versions between header and libs
|
|
UndefinedFunction = -2206 //< An undefined function
|
|
};
|
|
|
|
// Audio buffer data type
|
|
public enum AudioDataType
|
|
{
|
|
// Signed 16-bit integer mono audio stream
|
|
S16_Mono,
|
|
// Signed 16-bit integer stereo audio stream
|
|
S16_Stereo,
|
|
// Signed 32-bit float mono audio stream
|
|
F32_Mono,
|
|
// Signed 32-bit float stereo audio stream
|
|
F32_Stereo
|
|
};
|
|
|
|
// Various visemes
|
|
public enum Viseme
|
|
{
|
|
sil,
|
|
PP,
|
|
FF,
|
|
TH,
|
|
DD,
|
|
kk,
|
|
CH,
|
|
SS,
|
|
nn,
|
|
RR,
|
|
aa,
|
|
E,
|
|
ih,
|
|
oh,
|
|
ou
|
|
};
|
|
|
|
public static readonly int VisemeCount = Enum.GetNames(typeof(Viseme)).Length;
|
|
|
|
// Enum for sending lip-sync engine specific signals
|
|
public enum Signals
|
|
{
|
|
VisemeOn,
|
|
VisemeOff,
|
|
VisemeAmount,
|
|
VisemeSmoothing,
|
|
LaughterAmount
|
|
};
|
|
|
|
public static readonly int SignalCount = Enum.GetNames(typeof(Signals)).Length;
|
|
|
|
// Enum for provider context to create
|
|
public enum ContextProviders
|
|
{
|
|
Original,
|
|
Enhanced,
|
|
Enhanced_with_Laughter,
|
|
};
|
|
|
|
/// NOTE: Opaque typedef for lip-sync context is an unsigned int (uint)
|
|
|
|
/// Current phoneme frame results
|
|
[System.Serializable]
|
|
public class Frame
|
|
{
|
|
public void CopyInput(Frame input)
|
|
{
|
|
frameNumber = input.frameNumber;
|
|
frameDelay = input.frameDelay;
|
|
input.Visemes.CopyTo(Visemes, 0);
|
|
laughterScore = input.laughterScore;
|
|
}
|
|
|
|
public void Reset()
|
|
{
|
|
frameNumber = 0;
|
|
frameDelay = 0;
|
|
Array.Clear(Visemes, 0, VisemeCount);
|
|
laughterScore = 0;
|
|
}
|
|
|
|
public int frameNumber; // count from start of recognition
|
|
public int frameDelay; // in ms
|
|
public float[] Visemes = new float[VisemeCount]; // Array of floats for viseme frame. Size of Viseme Count, above
|
|
public float laughterScore; // probability of laughter presence.
|
|
};
|
|
|
|
// * * * * * * * * * * * * *
|
|
// Import functions
|
|
#if !UNITY_IOS || UNITY_EDITOR
|
|
public const string strOVRLS = "OVRLipSync";
|
|
#else
|
|
public const string strOVRLS = "__Internal";
|
|
#endif
|
|
[DllImport(strOVRLS)]
|
|
private static extern int ovrLipSyncDll_Initialize(int samplerate, int buffersize);
|
|
[DllImport(strOVRLS)]
|
|
private static extern void ovrLipSyncDll_Shutdown();
|
|
[DllImport(strOVRLS)]
|
|
private static extern IntPtr ovrLipSyncDll_GetVersion(ref int Major,
|
|
ref int Minor,
|
|
ref int Patch);
|
|
[DllImport(strOVRLS)]
|
|
private static extern int ovrLipSyncDll_CreateContextEx(ref uint context,
|
|
ContextProviders provider,
|
|
int sampleRate,
|
|
bool enableAcceleration);
|
|
|
|
[DllImport(strOVRLS)]
|
|
private static extern int ovrLipSyncDll_CreateContextWithModelFile(ref uint context,
|
|
ContextProviders provider,
|
|
string modelPath,
|
|
int sampleRate,
|
|
bool enableAcceleration);
|
|
|
|
[DllImport(strOVRLS)]
|
|
private static extern int ovrLipSyncDll_DestroyContext(uint context);
|
|
|
|
|
|
[DllImport(strOVRLS)]
|
|
private static extern int ovrLipSyncDll_ResetContext(uint context);
|
|
[DllImport(strOVRLS)]
|
|
private static extern int ovrLipSyncDll_SendSignal(uint context,
|
|
Signals signal,
|
|
int arg1, int arg2);
|
|
[DllImport(strOVRLS)]
|
|
private static extern int ovrLipSyncDll_ProcessFrameEx(
|
|
uint context,
|
|
IntPtr audioBuffer,
|
|
uint bufferSize,
|
|
AudioDataType dataType,
|
|
ref int frameNumber,
|
|
ref int frameDelay,
|
|
float[] visemes,
|
|
int visemeCount,
|
|
ref float laughterScore,
|
|
float[] laughterCategories,
|
|
int laughterCategoriesLength);
|
|
|
|
// * * * * * * * * * * * * *
|
|
// Public members
|
|
|
|
// * * * * * * * * * * * * *
|
|
// Static members
|
|
private static Result sInitialized = Result.Unknown;
|
|
|
|
// interface through this static member.
|
|
public static OVRLipSync sInstance = null;
|
|
|
|
|
|
// * * * * * * * * * * * * *
|
|
// MonoBehaviour overrides
|
|
|
|
/// <summary>
|
|
/// Awake this instance.
|
|
/// </summary>
|
|
void Awake()
|
|
{
|
|
// We can only have one instance of OVRLipSync in a scene (use this for local property query)
|
|
if (sInstance == null)
|
|
{
|
|
sInstance = this;
|
|
}
|
|
else
|
|
{
|
|
Debug.LogWarning(System.String.Format("OVRLipSync Awake: Only one instance of OVRPLipSync can exist in the scene."));
|
|
return;
|
|
}
|
|
|
|
if (IsInitialized() != Result.Success)
|
|
{
|
|
sInitialized = Initialize();
|
|
|
|
if (sInitialized != Result.Success)
|
|
{
|
|
Debug.LogWarning(System.String.Format
|
|
("OvrLipSync Awake: Failed to init Speech Rec library"));
|
|
}
|
|
}
|
|
|
|
// Important: Use the touchpad mechanism for input, call Create on the OVRTouchpad helper class
|
|
OVRTouchpad.Create();
|
|
|
|
}
|
|
|
|
/// <summary>
|
|
/// Raises the destroy event.
|
|
/// </summary>
|
|
void OnDestroy()
|
|
{
|
|
if (sInstance != this)
|
|
{
|
|
Debug.LogWarning(
|
|
"OVRLipSync OnDestroy: This is not the correct OVRLipSync instance.");
|
|
return;
|
|
}
|
|
|
|
// Do not shut down at this time
|
|
// ovrLipSyncDll_Shutdown();
|
|
// sInitialized = (int)Result.Unknown;
|
|
}
|
|
|
|
|
|
// * * * * * * * * * * * * *
|
|
// Public Functions
|
|
|
|
public static Result Initialize()
|
|
{
|
|
int sampleRate;
|
|
int bufferSize;
|
|
int numbuf;
|
|
|
|
// Get the current sample rate
|
|
sampleRate = AudioSettings.outputSampleRate;
|
|
// Get the current buffer size and number of buffers
|
|
AudioSettings.GetDSPBufferSize(out bufferSize, out numbuf);
|
|
|
|
String str = System.String.Format
|
|
("OvrLipSync Awake: Queried SampleRate: {0:F0} BufferSize: {1:F0}", sampleRate, bufferSize);
|
|
Debug.LogWarning(str);
|
|
|
|
sInitialized = (Result)ovrLipSyncDll_Initialize(sampleRate, bufferSize);
|
|
return sInitialized;
|
|
}
|
|
|
|
public static Result Initialize(int sampleRate, int bufferSize)
|
|
{
|
|
String str = System.String.Format
|
|
("OvrLipSync Awake: Queried SampleRate: {0:F0} BufferSize: {1:F0}", sampleRate, bufferSize);
|
|
Debug.LogWarning(str);
|
|
|
|
sInitialized = (Result)ovrLipSyncDll_Initialize(sampleRate, bufferSize);
|
|
return sInitialized;
|
|
}
|
|
|
|
public static void Shutdown()
|
|
{
|
|
ovrLipSyncDll_Shutdown();
|
|
sInitialized = Result.Unknown;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Determines if is initialized.
|
|
/// </summary>
|
|
/// <returns><c>true</c> if is initialized; otherwise, <c>false</c>.</returns>
|
|
public static Result IsInitialized()
|
|
{
|
|
return sInitialized;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a lip-sync context.
|
|
/// </summary>
|
|
/// <returns>error code</returns>
|
|
/// <param name="context">Context.</param>
|
|
/// <param name="provider">Provider.</param>
|
|
/// <param name="enableAcceleration">Enable DSP Acceleration.</param>
|
|
public static Result CreateContext(
|
|
ref uint context,
|
|
ContextProviders provider,
|
|
int sampleRate = 0,
|
|
bool enableAcceleration = false)
|
|
{
|
|
if (IsInitialized() != Result.Success && Initialize() != Result.Success)
|
|
return Result.CannotCreateContext;
|
|
|
|
return (Result)ovrLipSyncDll_CreateContextEx(ref context, provider, sampleRate, enableAcceleration);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a lip-sync context with specified model file.
|
|
/// </summary>
|
|
/// <returns>error code</returns>
|
|
/// <param name="context">Context.</param>
|
|
/// <param name="provider">Provider.</param>
|
|
/// <param name="modelPath">Model Dir.</param>
|
|
/// <param name="sampleRate">Sampling Rate.</param>
|
|
/// <param name="enableAcceleration">Enable DSP Acceleration.</param>
|
|
public static Result CreateContextWithModelFile(
|
|
ref uint context,
|
|
ContextProviders provider,
|
|
string modelPath,
|
|
int sampleRate = 0,
|
|
bool enableAcceleration = false)
|
|
{
|
|
if (IsInitialized() != Result.Success && Initialize() != Result.Success)
|
|
return Result.CannotCreateContext;
|
|
|
|
return (Result)ovrLipSyncDll_CreateContextWithModelFile(
|
|
ref context,
|
|
provider,
|
|
modelPath,
|
|
sampleRate,
|
|
enableAcceleration);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Destroy a lip-sync context.
|
|
/// </summary>
|
|
/// <returns>The context.</returns>
|
|
/// <param name="context">Context.</param>
|
|
public static Result DestroyContext(uint context)
|
|
{
|
|
if (IsInitialized() != Result.Success)
|
|
return Result.Unknown;
|
|
|
|
return (Result)ovrLipSyncDll_DestroyContext(context);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Resets the context.
|
|
/// </summary>
|
|
/// <returns>error code</returns>
|
|
/// <param name="context">Context.</param>
|
|
public static Result ResetContext(uint context)
|
|
{
|
|
if (IsInitialized() != Result.Success)
|
|
return Result.Unknown;
|
|
|
|
return (Result)ovrLipSyncDll_ResetContext(context);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Sends a signal to the lip-sync engine.
|
|
/// </summary>
|
|
/// <returns>error code</returns>
|
|
/// <param name="context">Context.</param>
|
|
/// <param name="signal">Signal.</param>
|
|
/// <param name="arg1">Arg1.</param>
|
|
/// <param name="arg2">Arg2.</param>
|
|
public static Result SendSignal(uint context, Signals signal, int arg1, int arg2)
|
|
{
|
|
if (IsInitialized() != Result.Success)
|
|
return Result.Unknown;
|
|
|
|
return (Result)ovrLipSyncDll_SendSignal(context, signal, arg1, arg2);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Process float[] audio buffer by lip-sync engine.
|
|
/// </summary>
|
|
/// <returns>error code</returns>
|
|
/// <param name="context">Context.</param>
|
|
/// <param name="audioBuffer"> PCM audio buffer.</param>
|
|
/// <param name="frame">Lip-sync Frame.</param>
|
|
/// <param name="stereo">Whether buffer is part of stereo or mono stream.</param>
|
|
public static Result ProcessFrame(
|
|
uint context, float[] audioBuffer, Frame frame, bool stereo = true)
|
|
{
|
|
if (IsInitialized() != Result.Success)
|
|
return Result.Unknown;
|
|
|
|
var dataType = stereo ? AudioDataType.F32_Stereo : AudioDataType.F32_Mono;
|
|
var numSamples = (uint)(stereo ? audioBuffer.Length / 2 : audioBuffer.Length);
|
|
var handle = GCHandle.Alloc(audioBuffer, GCHandleType.Pinned);
|
|
var rc = ovrLipSyncDll_ProcessFrameEx(context,
|
|
handle.AddrOfPinnedObject(), numSamples, dataType,
|
|
ref frame.frameNumber, ref frame.frameDelay,
|
|
frame.Visemes, frame.Visemes.Length,
|
|
ref frame.laughterScore,
|
|
null, 0
|
|
);
|
|
handle.Free();
|
|
return (Result)rc;
|
|
|
|
}
|
|
|
|
/// <summary>
|
|
/// Process short[] audio buffer by lip-sync engine.
|
|
/// </summary>
|
|
/// <returns>error code</returns>
|
|
/// <param name="context">Context.</param>
|
|
/// <param name="audioBuffer"> PCM audio buffer.</param>
|
|
/// <param name="frame">Lip-sync Frame.</param>
|
|
/// <param name="stereo">Whether buffer is part of stereo or mono stream.</param>
|
|
public static Result ProcessFrame(
|
|
uint context, short[] audioBuffer, Frame frame, bool stereo = true)
|
|
{
|
|
if (IsInitialized() != Result.Success)
|
|
return Result.Unknown;
|
|
|
|
var dataType = stereo ? AudioDataType.S16_Stereo : AudioDataType.S16_Mono;
|
|
var numSamples = (uint)(stereo ? audioBuffer.Length / 2 : audioBuffer.Length);
|
|
var handle = GCHandle.Alloc(audioBuffer, GCHandleType.Pinned);
|
|
var rc = ovrLipSyncDll_ProcessFrameEx(context,
|
|
handle.AddrOfPinnedObject(), numSamples, dataType,
|
|
ref frame.frameNumber, ref frame.frameDelay,
|
|
frame.Visemes, frame.Visemes.Length,
|
|
ref frame.laughterScore,
|
|
null, 0
|
|
);
|
|
handle.Free();
|
|
return (Result)rc;
|
|
}
|
|
|
|
}
|