Unity【Live Capture】- 关于人脸捕捉的解决方案

using System;
using System.IO;
using System.Linq;
using System.Text;
using UnityEngine;
using System.Collections.Generic;
using Unity.LiveCapture.Networking;
using Unity.LiveCapture.Networking.Discovery;

namespace Unity.LiveCapture.CompanionApp
{
    [CreateAssetMenu(menuName = "Live Capture/Server")]
    public class LiveCaptureServer : Server
    {
        const int k_DefaultPort = 9000;

        /// <summary>
        /// The server executes this event when a client has connected.
        /// </summary>
        public static event Action<ICompanionAppClient> ClientConnected = delegate { };

        /// <summary>
        /// The server executes this event when a client has disconnected.
        /// </summary>
        public static event Action<ICompanionAppClient> ClientDisconnected = delegate { };

        struct ConnectHandler
        {
            public string Name;
            public DateTime Time;
            public Func<ICompanionAppClient, bool> Handler;
        }

        readonly Dictionary<string, Type> s_TypeToClientType = new Dictionary<string, Type>();
        static readonly List<ConnectHandler> s_ClientConnectHandlers = new List<ConnectHandler>();

        /// <summary>
        /// Adds a callback used to take ownership of a client that has connected.
        /// </summary>
        /// <param name="handler">The callback function. It must return true if it takes ownership of a client.</param>
        /// <param name="name">The name of the client to prefer. If set, this handler has priority over clients that have the given name.</param>
        /// <param name="time">The time used to determine the priority of handlers when many are listening for the same
        /// client <paramref name="name"/>. More recent values have higher priority.</param>
        /// <exception cref="ArgumentNullException">Thrown if <paramref name="handler"/> is null.</exception>
        public static void RegisterClientConnectHandler(Func<ICompanionAppClient, bool> handler, string name, DateTime time)
        {
            if (handler == null)
                throw new ArgumentNullException(nameof(handler));

            DeregisterClientConnectHandler(handler);

            s_ClientConnectHandlers.Add(new ConnectHandler
            {
                Name = name,
                Time = time,
                Handler = handler,
            });
        }

        /// <summary>
        /// Removes a client connection callback.
        /// </summary>
        /// <param name="handler">The callback to remove.</param>>
        /// <exception cref="ArgumentNullException">Thrown if <paramref name="handler"/> is null.</exception>
        public static void DeregisterClientConnectHandler(Func<ICompanionAppClient, bool> handler)
        {
            if (handler == null)
                throw new ArgumentNullException(nameof(handler));

            for (var i = 0; i < s_ClientConnectHandlers.Count; i++)
            {
                if (s_ClientConnectHandlers[i].Handler == handler)
                {
                    s_ClientConnectHandlers.RemoveAt(i);
                }
            }
        }

        public void Init()
        {
            foreach (var (type, attributes) in AttributeUtility.GetAllTypes<ClientAttribute>())
            {
                if (!typeof(CompanionAppClient).IsAssignableFrom(type))
                {
                    Debug.LogError($"{type.FullName} must be assignable from {nameof(CompanionAppClient)} to use the {nameof(ClientAttribute)} attribute.");
                    continue;
                }

                foreach (var attribute in attributes)
                {
                    s_TypeToClientType[attribute.Type] = type;
                }
            }
        }

        [SerializeField, Tooltip("The TCP port on which the server will listen for incoming connections. Changes to the port only take effect after restarting the server.")]
        int m_Port = k_DefaultPort;
        [SerializeField, Tooltip("Start the server automatically after entering play mode.")]
        bool m_AutoStartOnPlay = true;

        readonly DiscoveryServer m_Discovery = new DiscoveryServer();
        readonly NetworkServer m_Server = new NetworkServer();
        readonly Dictionary<Remote, ICompanionAppClient> m_RemoteToClient = new Dictionary<Remote, ICompanionAppClient>();

        /// <summary>
        /// The TCP port on which the server will listen for incoming connections.
        /// </summary>
        /// <remarks>
        /// Changes to the port only take effect after restarting the server.
        /// </remarks>
        public int Port
        {
            get => m_Port;
            set
            {
                if (m_Port != value)
                {
                    m_Port = value;
                    OnServerChanged(true);
                }
            }
        }

        /// <summary>
        /// Start the server automatically after entering play mode.
        /// </summary>
        public bool AutoStartOnPlay
        {
            get => m_AutoStartOnPlay;
            set
            {
                if (m_AutoStartOnPlay != value)
                {
                    m_AutoStartOnPlay = value;
                    OnServerChanged(true);
                }
            }
        }

        /// <summary>
        /// Are clients able to connect to the server.
        /// </summary>
        public bool IsRunning => m_Server.IsRunning;

        /// <summary>
        /// The number of clients currently connected to the server.
        /// </summary>
        public int ClientCount => m_RemoteToClient.Count;

        /// <inheritdoc/>
        protected override void OnEnable()
        {
            base.OnEnable();

            m_Server.RemoteConnected += OnClientConnected;
            m_Server.RemoteDisconnected += OnClientDisconnected;
        }

        /// <inheritdoc/>
        protected override void OnDisable()
        {
            base.OnDisable();

            m_Discovery.Stop();
            m_Server.Stop();

            m_Server.RemoteConnected -= OnClientConnected;
            m_Server.RemoteDisconnected -= OnClientDisconnected;
        }

        /// <summary>
        /// Gets the currently connected clients.
        /// </summary>
        /// <returns>A new collection containing the client handles.</returns>
        public IEnumerable<ICompanionAppClient> GetClients()
        {
            return m_RemoteToClient.Values;
        }

        /// <inheritdoc />
        public override string GetName() => "Companion App Server";

        /// <summary>
        /// Start listening for clients connections.
        /// </summary>
        public void StartServer()
        {
            if (!NetworkUtilities.IsPortAvailable(m_Port))
            {
                Debug.LogError($"Unable to start server: Port {m_Port} is in use by another program! Close the other program, or assign a free port using the Live Capture Window.");
                return;
            }

            if (m_Server.StartServer(m_Port))
            {
                // start server discovery
                var config = new ServerData(
                    "Live Capture",
                    Environment.MachineName,
                    m_Server.ID,
                    PackageUtility.GetVersion(LiveCaptureInfo.Version)
                );
                var endPoints = m_Server.EndPoints.ToArray();

                m_Discovery.Start(config, endPoints);
            }

            OnServerChanged(false);
        }

        /// <summary>
        /// Disconnects all clients and stop listening for new connections.
        /// </summary>
        public void StopServer()
        {
            m_Server.Stop();
            m_Discovery.Stop();

            OnServerChanged(false);
        }

        /// <inheritdoc/>
        public override void OnUpdate()
        {
            m_Server.Update();
            m_Discovery.Update();
        }

        void OnClientConnected(Remote remote)
        {
            m_Server.RegisterMessageHandler(remote, InitializeClient, false);
        }

        void OnClientDisconnected(Remote remote, DisconnectStatus status)
        {
            if (m_RemoteToClient.TryGetValue(remote, out var client))
            {
                try
                {
                    ClientDisconnected.Invoke(client);
                }
                catch (Exception e)
                {
                    Debug.LogError(e);
                }

                m_RemoteToClient.Remove(remote);
                OnServerChanged(false);
            }
        }

        void InitializeClient(Message message)
        {
            try
            {
                if (message.ChannelType != ChannelType.ReliableOrdered)
                {
                    return;
                }

                var streamReader = new StreamReader(message.Data, Encoding.UTF8);
                var json = streamReader.ReadToEnd();
                var data = default(ClientInitialization);

                try
                {
                    data = JsonUtility.FromJson<ClientInitialization>(json);
                }
                catch (Exception)
                {
                    Debug.LogError($"{nameof(CompanionAppServer)} failed to initialize client connection! Could not parse JSON: {json}");
                    return;
                }

                if (!s_TypeToClientType.TryGetValue(data.Type, out var clientType))
                {
                    Debug.LogError($"Unknown client type \"{data.Type}\" connected to {nameof(CompanionAppServer)}!");
                    return;
                }

                var remote = message.Remote;
                var client = Activator.CreateInstance(clientType, m_Server, remote, data) as CompanionAppClient;
                client.SendProtocol();

                m_RemoteToClient.Add(remote, client);

                AssignOwner(client);

                ClientConnected.Invoke(client);
                OnServerChanged(false);
            }
            catch (Exception e)
            {
                Debug.LogException(e);
            }
            finally
            {
                message.Dispose();
            }
        }

        void AssignOwner(ICompanionAppClient client)
        {
            // connect to the registered handler that was most recently used with this client if possible
            foreach (var handler in s_ClientConnectHandlers.OrderByDescending(h => h.Time.Ticks))
            {
                try
                {
                    if (handler.Name == client.Name)
                    {
                        if (handler.Handler(client))
                            return;
                    }
                }
                catch (Exception e)
                {
                    Debug.LogException(e);
                }
            }

            // fall back to the first free device that is compatible with the client
            foreach (var handler in s_ClientConnectHandlers)
            {
                try
                {
                    if (handler.Handler(client))
                        return;
                }
                catch (Exception e)
                {
                    Debug.LogException(e);
                }
            }
        }
    }
}

using System;
using UnityEngine;

namespace Unity.LiveCapture.CompanionApp
{
    /// <summary>
    /// A type of <see cref="LiveCaptureDevice"/> that uses a <see cref="ICompanionAppClient"/> for communication.
    /// </summary>
    interface ICompanionAppDevice
    {
        /// <summary>
        /// Clears the client assigned to this device.
        /// </summary>
        void ClearClient();
    }

    /// <summary>
    /// A type of <see cref="LiveCaptureDevice"/> that uses a <see cref="ICompanionAppClient"/> for communication.
    /// </summary>
    /// <typeparam name="TClient">The type of client this device communicates with.</typeparam>
    public abstract class CompanionAppDevice<TClient> : LiveCaptureDevice, ICompanionAppDevice where TClient : class, ICompanionAppClient
    {
        bool m_ClientRegistered;
        bool m_Recording;
        TClient m_Client;
        readonly SlateChangeTracker m_SlateChangeTracker = new SlateChangeTracker();
        readonly TakeNameFormatter m_TakeNameFormatter = new TakeNameFormatter();
        string m_LastAssetName;

        bool TryGetInternalClient(out ICompanionAppClientInternal client)
{
            client = m_Client as ICompanionAppClientInternal;

            return client != null;
        }

        /// <summary>
        /// This function is called when the object becomes enabled and active.
        /// </summary>
        protected virtual void OnEnable()
{
            CompanionAppServer.ClientDisconnected += OnClientDisconnected;
            LiveCaptureServer.ClientDisconnected += OnClientDisconnected;
            RegisterClient();
        }

        /// <summary>
        /// This function is called when the behaviour becomes disabled.
        /// </summary>
        /// <remaks>
        /// This is also called when the object is destroyed and can be used for any cleanup code.
        ///  When scripts are reloaded after compilation has finished, OnDisable will be called, followed by an OnEnable after the script has been loaded.
        /// </remaks>
        protected virtual void OnDisable()
{
            CompanionAppServer.ClientDisconnected -= OnClientDisconnected;
            CompanionAppServer.DeregisterClientConnectHandler(OnClientConnected);
            LiveCaptureServer.ClientConnected -= OnClientDisconnected;
            LiveCaptureServer.DeregisterClientConnectHandler(OnClientConnected);

            StopRecording();
            UnregisterClient();
        }

        /// <summary>
        /// This function is called when the behaviour gets destroyed.
        /// </summary>
        protected override void OnDestroy()
{
            base.OnDestroy();

            ClearClient();
        }

        /// <inheritdoc/>
        public override bool IsReady()
{
            return m_Client != null;
        }

        /// <inheritdoc/>
        public override bool IsRecording()
{
            return m_Recording;
        }

        /// <inheritdoc/>
        public override void StartRecording()
{
            if (!m_Recording)
            {
                m_Recording = true;

                OnRecordingChanged();
                SendRecordingState();
            }
        }

        /// <inheritdoc/>
        public override void StopRecording()
{
            if (m_Recording)
            {
                m_Recording = false;

                OnRecordingChanged();
                SendRecordingState();
            }
        }

        /// <summary>
        /// Gets the client currently assigned to this device.
        /// </summary>
        /// <returns>The assigned client, or null if none is assigned.</returns>
        public TClient GetClient()
{
            return m_Client;
        }

        /// <summary>
        /// Assigns a client to this device.
        /// </summary>
        /// <param name="client">The client to assign, or null to clear the assigned client.</param>
        /// <param name="rememberAssignment">Try to auto-assign the client to this device when it reconnects in the future.</param>
        public void SetClient(TClient client, bool rememberAssignment)
{
            if (m_Client != client)
            {
                UnregisterClient();

                if (m_Client != null)
                {
                    ClientMappingDatabase.DeregisterClientAssociation(this, m_Client, rememberAssignment);
                }

                m_Client = client;

                if (m_Client != null)
                {
                    // if any device is also using this client, we must clear the client from the previous device.
                    if (ClientMappingDatabase.TryGetDevice(client, out var previousDevice))
                    {
                        previousDevice.ClearClient();
                    }

                    ClientMappingDatabase.RegisterClientAssociation(this, m_Client, rememberAssignment);
                }

                RegisterClient();
            }
        }

        void RegisterClient()
{
            if (!isActiveAndEnabled ||  m_ClientRegistered)
            {
                return;
            }

            LiveCaptureServer.DeregisterClientConnectHandler(OnClientConnected);
            CompanionAppServer.DeregisterClientConnectHandler(OnClientConnected);

            m_SlateChangeTracker.Reset();

            if (TryGetInternalClient(out var client))
            {
                client.SetDeviceMode += ClientSetDeviceMode;
                client.StartRecording += ClientStartRecording;
                client.StopRecording += ClientStopRecording;
                client.StartPlayer += ClientStartPlayer;
                client.StopPlayer += ClientStopPlayer;
                client.PausePlayer += ClientPausePlayer;
                client.SetPlayerTime += ClientSetPlayerTime;
                client.SetSelectedTake += ClientSetSelectedTake;
                client.SetTakeData += ClientSetTakeData;
                client.DeleteTake += ClientDeleteTake;
                client.SetIterationBase += ClientSetIterationBase;
                client.ClearIterationBase += ClientClearIterationBase;
                client.TexturePreviewRequested += OnTexturePreviewRequested;

                OnClientAssigned();

                client.SendInitialize();

                UpdateClient();

                 m_ClientRegistered = true;
            }
            else
            {
                ClientMappingDatabase.TryGetClientAssignment(this, out var clientName, out var time);
                LiveCaptureServer.RegisterClientConnectHandler(OnClientConnected, clientName, time);
                CompanionAppServer.RegisterClientConnectHandler(OnClientConnected, clientName, time);
            }
        }

        void UnregisterClient()
{
            if (!m_ClientRegistered)
            {
                return;
            }

            if (TryGetInternalClient(out var client))
            {
                OnClientUnassigned();

                client.SendEndSession();
                client.SetDeviceMode -= ClientSetDeviceMode;
                client.StartRecording -= ClientStartRecording;
                client.StopRecording -= ClientStopRecording;
                client.StartPlayer -= ClientStartPlayer;
                client.StopPlayer -= ClientStopPlayer;
                client.PausePlayer -= ClientPausePlayer;
                client.SetPlayerTime -= ClientSetPlayerTime;
                client.SetSelectedTake -= ClientSetSelectedTake;
                client.SetTakeData -= ClientSetTakeData;
                client.DeleteTake -= ClientDeleteTake;
                client.SetIterationBase -= ClientSetIterationBase;
                client.ClearIterationBase -= ClientClearIterationBase;
                client.TexturePreviewRequested -= OnTexturePreviewRequested;

                m_ClientRegistered = false;
            }
        }

        /// <inheritdoc />
        public void ClearClient()
{
            SetClient(null, true);
        }

        /// <summary>
        /// Called to send the device state to the client.
        /// </summary>
        public virtual void UpdateClient()
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                SendDeviceState(takeRecorder.IsLive());

                var slate = takeRecorder.GetActiveSlate();
                var hasSlate = slate != null;
                var slateChanged = m_SlateChangeTracker.Changed(slate);
                var take = hasSlate ? slate.Take : null;

                var assetName = GetAssetName();
                var assetNameChanged = assetName != m_LastAssetName;
                m_LastAssetName = assetName;

                if (TryGetInternalClient(out var client))
                {
                    client.SendFrameRate(takeRecorder.IsLive() || take == null ? takeRecorder.FrameRate : take.FrameRate);
                    client.SendHasSlate(hasSlate);
                    client.SendSlateDuration(hasSlate ? slate.Duration : 0d);
                    client.SendSlateIsPreviewing(takeRecorder.IsPreviewPlaying());
                    client.SendSlatePreviewTime(takeRecorder.GetPreviewTime());

                    if (slateChanged || assetNameChanged)
                    {
                        if (hasSlate)
                            m_TakeNameFormatter.ConfigureTake(slate.SceneNumber, slate.ShotName, slate.TakeNumber);
                        else
                            m_TakeNameFormatter.ConfigureTake(0, "Shot", 0);

                        client.SendNextTakeName(m_TakeNameFormatter.GetTakeName());
                        client.SendNextAssetName(m_TakeNameFormatter.GetAssetName());
                    }
                }

                if (slateChanged)
                {
                    SendSlateDescriptor(slate);
                }
            }

            SendRecordingState();
        }

        /// <summary>
        /// Gets the name used for the take asset name.
        /// </summary>
        /// <returns>The name of the asset.</returns>
        protected virtual string GetAssetName() { return name; }

        /// <summary>
        /// The device calls this method when a new client is assigned.
        /// </summary>
        protected virtual void OnClientAssigned() {}

        /// <summary>
        /// The device calls this method when the client is unassigned.
        /// </summary>
        protected virtual void OnClientUnassigned() {}

        /// <summary>
        /// The device calls this method when the recording state has changed.
        /// </summary>
        protected virtual void OnRecordingChanged() {}

        /// <summary>
        /// The device calls this method when the slate has changed.
        /// </summary>
        protected virtual void OnSlateChanged(ISlate slate) {}

        void ClientStartRecording()
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                takeRecorder.StartRecording();
            }

            Refresh();
        }

        void ClientStopRecording()
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                takeRecorder.StopRecording();
            }

            Refresh();
        }

        void ClientSetDeviceMode(DeviceMode deviceMode)
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                takeRecorder.SetLive(deviceMode == DeviceMode.LiveStream);

                SendDeviceState(takeRecorder.IsLive());
            }
        }

        void ClientStartPlayer()
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                takeRecorder.PlayPreview();
            }

            Refresh();
        }

        void ClientStopPlayer()
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                takeRecorder.PausePreview();
                takeRecorder.SetPreviewTime(0d);
            }

            Refresh();
        }

        void ClientPausePlayer()
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                takeRecorder.PausePreview();
            }

            Refresh();
        }

        void ClientSetPlayerTime(double time)
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                takeRecorder.SetPreviewTime(time);
            }

            Refresh();
        }

        void SendDeviceState()
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                SendDeviceState(takeRecorder.IsLive());
            }
        }

        void SendDeviceState(bool isLive)
{
            if (TryGetInternalClient(out var client))
            {
                client.SendDeviceMode(isLive ? DeviceMode.LiveStream : DeviceMode.Playback);
            }
        }

        void SendRecordingState()
{
            if (TryGetInternalClient(out var client))
            {
                client.SendRecordingState(IsRecording());
            }
        }

        void SendSlateDescriptor()
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                SendSlateDescriptor(takeRecorder.GetActiveSlate());
            }
        }

        void SendSlateDescriptor(ISlate slate)
{
            if (TryGetInternalClient(out var client))
            {
                client.SendSlateDescriptor(SlateDescriptor.Create(slate));
            }

            OnSlateChanged(slate);
        }

        void ClientSetSelectedTake(Guid guid)
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                TakeManager.Default.SelectTake(takeRecorder.GetActiveSlate(), guid);

                SendSlateDescriptor();
                Refresh();
            }
        }

        void ClientSetTakeData(TakeDescriptor descriptor)
{
            TakeManager.Default.SetTakeData(descriptor);

            SendSlateDescriptor();
            Refresh();
        }

        void ClientDeleteTake(Guid guid)
{
            TakeManager.Default.DeleteTake(guid);

            SendSlateDescriptor();
            Refresh();
        }

        void ClientSetIterationBase(Guid guid)
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                var slate = takeRecorder.GetActiveSlate();

                TakeManager.Default.SetIterationBase(slate, guid);

                SendSlateDescriptor(slate);
                Refresh();
            }
        }

        void ClientClearIterationBase()
{
            var takeRecorder = GetTakeRecorder();

            if (takeRecorder != null)
            {
                var slate = takeRecorder.GetActiveSlate();

                TakeManager.Default.ClearIterationBase(slate);

                SendSlateDescriptor(slate);
                Refresh();
            }
        }

        void OnTexturePreviewRequested(Guid guid)
{
            var texture = TakeManager.Default.GetAssetPreview<Texture2D>(guid);

            if (texture != null && TryGetInternalClient(out var client))
            {
                client.SendTexturePreview(guid, texture);
            }
        }

        bool OnClientConnected(ICompanionAppClient client)
{
            if (m_Client == null && client is TClient c && (!ClientMappingDatabase.TryGetClientAssignment(this, out var clientName, out _) || c.Name == clientName))
            {
                SetClient(c, false);
                return true;
            }
            return false;
        }

        void OnClientDisconnected(ICompanionAppClient client)
{
            if (m_Client == client)
            {
                SetClient(null, false);
            }
        }
    }
}

using UnityEngine;
using Unity.LiveCapture.CompanionApp;

public class LiveCaptureExample : MonoBehaviour
{
    LiveCaptureServer server;
    private void Awake()
    {
        server = Resources.Load<LiveCaptureServer>("Live Capture Server");
        server.Init();
        server.StartServer();
    }
    private void Update()
    {
        server.OnUpdate();
    }
    private void OnDestroy()
    {
        server.StopServer();
    }
}