WitService.cs 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787
  1. /*
  2. * Copyright (c) Meta Platforms, Inc. and affiliates.
  3. * All rights reserved.
  4. *
  5. * This source code is licensed under the license found in the
  6. * LICENSE file in the root directory of this source tree.
  7. */
  8. using UnityEngine;
  9. using System.Collections;
  10. using System.Collections.Generic;
  11. using Meta.Voice;
  12. using Meta.WitAi.Configuration;
  13. using Meta.WitAi.Data;
  14. using Meta.WitAi.Data.Configuration;
  15. using Meta.WitAi.Events;
  16. using Meta.WitAi.Interfaces;
  17. using Meta.WitAi.Json;
  18. using Meta.WitAi.Requests;
  19. using UnityEngine.Events;
  20. using UnityEngine.SceneManagement;
  21. namespace Meta.WitAi
  22. {
  23. public class WitService : MonoBehaviour, IVoiceEventProvider, IVoiceActivationHandler, ITelemetryEventsProvider, IWitRuntimeConfigProvider, IWitConfigurationProvider
  24. {
  25. private float _lastMinVolumeLevelTime;
  26. private WitRequest _recordingRequest;
  27. private bool _isSoundWakeActive;
  28. private RingBuffer<byte>.Marker _lastSampleMarker;
  29. private bool _minKeepAliveWasHit;
  30. private bool _isActive;
  31. private long _minSampleByteCount = 1024 * 10;
  32. private IVoiceEventProvider _voiceEventProvider;
  33. private ITelemetryEventsProvider _telemetryEventsProvider;
  34. private IWitRuntimeConfigProvider _runtimeConfigProvider;
  35. private ITranscriptionProvider _activeTranscriptionProvider;
  36. private Coroutine _timeLimitCoroutine;
  37. private IWitRequestProvider _witRequestProvider;
  38. // Transcription based endpointing
  39. private bool _receivedTranscription;
  40. private float _lastWordTime;
  41. // Parallel Requests
  42. private HashSet<VoiceServiceRequest> _transmitRequests = new HashSet<VoiceServiceRequest>();
  43. private Coroutine _queueHandler;
  44. // Wit configuration provider
  45. public WitConfiguration Configuration => RuntimeConfiguration?.witConfiguration;
  46. #region Interfaces
  47. private IWitByteDataReadyHandler[] _dataReadyHandlers;
  48. private IWitByteDataSentHandler[] _dataSentHandlers;
  49. private IDynamicEntitiesProvider[] _dynamicEntityProviders;
  50. private float _time;
  51. #endregion
  52. /// <summary>
  53. /// Returns true if wit is currently active and listening with the mic
  54. /// </summary>
  55. public bool Active => _isActive || IsRequestActive;
  56. /// <summary>
  57. /// Active if recording, transmitting, or queued up
  58. /// </summary>
  59. public bool IsRequestActive
  60. {
  61. get
  62. {
  63. if (null != _recordingRequest && _recordingRequest.IsActive)
  64. {
  65. return true;
  66. }
  67. return false;
  68. }
  69. }
  70. public IVoiceEventProvider VoiceEventProvider
  71. {
  72. get => _voiceEventProvider;
  73. set => _voiceEventProvider = value;
  74. }
  75. public ITelemetryEventsProvider TelemetryEventsProvider
  76. {
  77. get => _telemetryEventsProvider;
  78. set => _telemetryEventsProvider = value;
  79. }
  80. public IWitRuntimeConfigProvider ConfigurationProvider
  81. {
  82. get => _runtimeConfigProvider;
  83. set => _runtimeConfigProvider = value;
  84. }
  85. public WitRuntimeConfiguration RuntimeConfiguration =>
  86. _runtimeConfigProvider?.RuntimeConfiguration;
  87. public VoiceEvents VoiceEvents => _voiceEventProvider.VoiceEvents;
  88. public TelemetryEvents TelemetryEvents => _telemetryEventsProvider.TelemetryEvents;
  89. /// <summary>
  90. /// Gets/Sets a custom transcription provider. This can be used to replace any built in asr
  91. /// with an on device model or other provided source
  92. /// </summary>
  93. public ITranscriptionProvider TranscriptionProvider
  94. {
  95. get => _activeTranscriptionProvider;
  96. set
  97. {
  98. if (null != _activeTranscriptionProvider)
  99. {
  100. _activeTranscriptionProvider.OnFullTranscription.RemoveListener(
  101. OnFullTranscription);
  102. _activeTranscriptionProvider.OnPartialTranscription.RemoveListener(
  103. OnPartialTranscription);
  104. _activeTranscriptionProvider.OnMicLevelChanged.RemoveListener(
  105. OnTranscriptionMicLevelChanged);
  106. _activeTranscriptionProvider.OnStartListening.RemoveListener(
  107. OnMicStartListening);
  108. _activeTranscriptionProvider.OnStoppedListening.RemoveListener(
  109. OnMicStoppedListening);
  110. }
  111. _activeTranscriptionProvider = value;
  112. if (null != _activeTranscriptionProvider)
  113. {
  114. _activeTranscriptionProvider.OnFullTranscription.AddListener(
  115. OnFullTranscription);
  116. _activeTranscriptionProvider.OnPartialTranscription.AddListener(
  117. OnPartialTranscription);
  118. _activeTranscriptionProvider.OnMicLevelChanged.AddListener(
  119. OnTranscriptionMicLevelChanged);
  120. _activeTranscriptionProvider.OnStartListening.AddListener(
  121. OnMicStartListening);
  122. _activeTranscriptionProvider.OnStoppedListening.AddListener(
  123. OnMicStoppedListening);
  124. }
  125. }
  126. }
  127. public IWitRequestProvider WitRequestProvider
  128. {
  129. get => _witRequestProvider;
  130. set => _witRequestProvider = value;
  131. }
  132. public bool MicActive => AudioBuffer.Instance.IsRecording(this);
  133. protected bool ShouldSendMicData => RuntimeConfiguration.sendAudioToWit ||
  134. null == _activeTranscriptionProvider;
  135. /// <summary>
  136. /// Check configuration, client access token & app id
  137. /// </summary>
  138. public virtual bool IsConfigurationValid()
  139. {
  140. return RuntimeConfiguration.witConfiguration != null &&
  141. !string.IsNullOrEmpty(RuntimeConfiguration.witConfiguration.GetClientAccessToken());
  142. }
  143. #region LIFECYCLE
  144. // Find transcription provider & Mic
  145. protected void Awake()
  146. {
  147. _dataReadyHandlers = GetComponents<IWitByteDataReadyHandler>();
  148. _dataSentHandlers = GetComponents<IWitByteDataSentHandler>();
  149. }
  150. // Add mic delegates
  151. protected void OnEnable()
  152. {
  153. SceneManager.sceneLoaded += OnSceneLoaded;
  154. _runtimeConfigProvider = GetComponent<IWitRuntimeConfigProvider>();
  155. _voiceEventProvider = GetComponent<IVoiceEventProvider>();
  156. if (null == _activeTranscriptionProvider && null != RuntimeConfiguration &&
  157. RuntimeConfiguration.customTranscriptionProvider)
  158. {
  159. TranscriptionProvider = RuntimeConfiguration.customTranscriptionProvider;
  160. }
  161. SetMicDelegates(true);
  162. _dynamicEntityProviders = GetComponents<IDynamicEntitiesProvider>();
  163. }
  164. // Remove mic delegates
  165. protected void OnDisable()
  166. {
  167. SceneManager.sceneLoaded -= OnSceneLoaded;
  168. AudioBufferEvents e = AudioBuffer.Instance?.Events;
  169. SetMicDelegates(false);
  170. }
  171. // On scene refresh
  172. protected virtual void OnSceneLoaded(Scene scene, LoadSceneMode mode)
  173. {
  174. SetMicDelegates(true);
  175. }
  176. // Toggle audio events
  177. private AudioBuffer _buffer;
  178. private bool _bufferDelegates = false;
  179. protected void SetMicDelegates(bool add)
  180. {
  181. // Obtain buffer
  182. if (_buffer == null)
  183. {
  184. _buffer = AudioBuffer.Instance;
  185. _bufferDelegates = false;
  186. }
  187. // Get events if possible
  188. AudioBufferEvents e = _buffer?.Events;
  189. if (e == null)
  190. {
  191. return;
  192. }
  193. // Already set
  194. if (_bufferDelegates == add)
  195. {
  196. return;
  197. }
  198. // Set delegates
  199. _bufferDelegates = add;
  200. // Add delegates
  201. if (add)
  202. {
  203. e.OnMicLevelChanged.AddListener(OnMicLevelChanged);
  204. e.OnByteDataReady.AddListener(OnByteDataReady);
  205. e.OnSampleReady += OnMicSampleReady;
  206. }
  207. // Remove delegates
  208. else
  209. {
  210. e.OnMicLevelChanged.RemoveListener(OnMicLevelChanged);
  211. e.OnByteDataReady.RemoveListener(OnByteDataReady);
  212. e.OnSampleReady -= OnMicSampleReady;
  213. }
  214. }
  215. #endregion
  216. #region ACTIVATION
  217. /// <summary>
  218. /// Activate the microphone and send data to Wit for NLU processing.
  219. /// </summary>
  220. public void Activate() => Activate(new WitRequestOptions());
  221. public void Activate(WitRequestOptions requestOptions) => Activate(requestOptions, new VoiceServiceRequestEvents());
  222. public VoiceServiceRequest Activate(WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents)
  223. {
  224. // Not valid
  225. if (!IsConfigurationValid())
  226. {
  227. VLog.E($"Your AppVoiceExperience \"{gameObject.name}\" does not have a wit config assigned. Understanding Viewer activations will not trigger in game events..");
  228. return null;
  229. }
  230. // Already recording
  231. if (_isActive)
  232. {
  233. return null;
  234. }
  235. // Stop recording
  236. StopRecording();
  237. // Now active
  238. _isActive = true;
  239. _lastSampleMarker = AudioBuffer.Instance.CreateMarker(ConfigurationProvider.RuntimeConfiguration.preferredActivationOffset);
  240. _lastMinVolumeLevelTime = float.PositiveInfinity;
  241. _lastWordTime = float.PositiveInfinity;
  242. _receivedTranscription = false;
  243. // Generate request
  244. WitRequest request = WitRequestProvider != null ? WitRequestProvider.CreateWitRequest(RuntimeConfiguration.witConfiguration, requestOptions, requestEvents, _dynamicEntityProviders)
  245. : RuntimeConfiguration.witConfiguration.CreateSpeechRequest(requestOptions, requestEvents, _dynamicEntityProviders);
  246. SetupRequest(request);
  247. // Start recording if possible
  248. if (ShouldSendMicData)
  249. {
  250. if (!AudioBuffer.Instance.IsRecording(this))
  251. {
  252. _minKeepAliveWasHit = false;
  253. _isSoundWakeActive = true;
  254. StartRecording();
  255. }
  256. _recordingRequest.ActivateAudio();
  257. }
  258. // Activate transcription provider
  259. _activeTranscriptionProvider?.Activate();
  260. // Return the generated request
  261. return _recordingRequest;
  262. }
  263. /// <summary>
  264. /// Activate the microphone and immediately send data to Wit for NLU processing.
  265. /// </summary>
  266. public void ActivateImmediately() => ActivateImmediately(new WitRequestOptions());
  267. public void ActivateImmediately(WitRequestOptions requestOptions) => ActivateImmediately(requestOptions, new VoiceServiceRequestEvents());
  268. public VoiceServiceRequest ActivateImmediately(WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents)
  269. {
  270. // Activate mic & generate request if possible
  271. var request = Activate(requestOptions, requestEvents);
  272. if (request == null)
  273. {
  274. return null;
  275. }
  276. // Send recording request
  277. SendRecordingRequest();
  278. // Start marker
  279. _lastSampleMarker = AudioBuffer.Instance.CreateMarker(ConfigurationProvider
  280. .RuntimeConfiguration.preferredActivationOffset);
  281. // Return the request
  282. return request;
  283. }
  284. /// <summary>
  285. /// Sends recording request if possible
  286. /// </summary>
  287. protected virtual void SendRecordingRequest()
  288. {
  289. if (_recordingRequest == null || _recordingRequest.State != VoiceRequestState.Initialized)
  290. {
  291. return;
  292. }
  293. // Sound wake active
  294. _isSoundWakeActive = false;
  295. // Execute request
  296. if (ShouldSendMicData)
  297. {
  298. ExecuteRequest(_recordingRequest);
  299. }
  300. }
  301. /// <summary>
  302. /// Setup recording request
  303. /// </summary>
  304. /// <param name="recordingRequest"></param>
  305. protected void SetupRequest(WitRequest newRequest)
  306. {
  307. if (_recordingRequest == newRequest)
  308. {
  309. return;
  310. }
  311. // Set request & events
  312. _recordingRequest = newRequest;
  313. _recordingRequest.Events.OnCancel.AddListener(HandleResult);
  314. _recordingRequest.Events.OnFailed.AddListener(HandleResult);
  315. _recordingRequest.Events.OnSuccess.AddListener(HandleResult);
  316. _recordingRequest.Events.OnComplete.AddListener(HandleComplete);
  317. // Call service events
  318. VoiceEvents.OnRequestOptionSetup?.Invoke(_recordingRequest.Options);
  319. VoiceEvents.OnRequestInitialized?.Invoke(_recordingRequest);
  320. }
  321. /// <summary>
  322. /// Execute a wit request immediately
  323. /// </summary>
  324. /// <param name="recordingRequest"></param>
  325. public void ExecuteRequest(WitRequest newRequest)
  326. {
  327. SetupRequest(newRequest);
  328. newRequest.AudioEncoding = AudioBuffer.Instance.AudioEncoding;
  329. newRequest.audioDurationTracker = new AudioDurationTracker(_recordingRequest.Options?.RequestId,
  330. newRequest.AudioEncoding);
  331. newRequest.onInputStreamReady += r => OnWitReadyForData();
  332. _recordingRequest.Events.OnPartialTranscription.AddListener(OnPartialTranscription);
  333. _recordingRequest.Events.OnFullTranscription.AddListener(OnFullTranscription);
  334. _recordingRequest.Events.OnPartialResponse.AddListener(HandlePartialResult);
  335. #pragma warning disable CS0618
  336. VoiceEvents.OnRequestCreated?.Invoke(_recordingRequest);
  337. VoiceEvents.OnSend?.Invoke(_recordingRequest);
  338. _timeLimitCoroutine = StartCoroutine(DeactivateDueToTimeLimit());
  339. _recordingRequest.Send();
  340. }
  341. #endregion
  342. #region TEXT REQUESTS
  343. /// <summary>
  344. /// Activate the microphone and send data to Wit for NLU processing.
  345. /// </summary>
  346. public void Activate(string text) => Activate(text, new WitRequestOptions());
  347. public void Activate(string text, WitRequestOptions requestOptions) => Activate(text, requestOptions, new VoiceServiceRequestEvents());
  348. public VoiceServiceRequest Activate(string text, WitRequestOptions requestOptions, VoiceServiceRequestEvents requestEvents)
  349. {
  350. // Not valid
  351. if (!IsConfigurationValid())
  352. {
  353. VLog.E($"Your AppVoiceExperience \"{gameObject.name}\" does not have a wit config assigned. Understanding Viewer activations will not trigger in game events..");
  354. return null;
  355. }
  356. // Handle option setup
  357. VoiceEvents.OnRequestOptionSetup?.Invoke(requestOptions);
  358. // Generate request
  359. VoiceServiceRequest request = Configuration.CreateMessageRequest(requestOptions, requestEvents, _dynamicEntityProviders);
  360. request.Events.OnCancel.AddListener(HandleResult);
  361. request.Events.OnFailed.AddListener(HandleResult);
  362. request.Events.OnSuccess.AddListener(HandleResult);
  363. request.Events.OnComplete.AddListener(HandleComplete);
  364. _transmitRequests.Add(request);
  365. // Call on create delegates
  366. VoiceEvents?.OnRequestInitialized?.Invoke(request);
  367. #pragma warning disable CS0618
  368. VoiceEvents?.OnRequestCreated?.Invoke(null);
  369. VoiceEvents?.OnSend?.Invoke(request);
  370. // Send & return
  371. request.Send(text);
  372. return request;
  373. }
  374. #endregion TEXT REQUESTS
  375. #region RECORDING
  376. // Stop any recording
  377. private void StopRecording()
  378. {
  379. if (!AudioBuffer.Instance.IsRecording(this)) return;
  380. AudioBuffer.Instance.StopRecording(this);
  381. }
  382. // When wit is ready, start recording
  383. private void OnWitReadyForData()
  384. {
  385. _lastMinVolumeLevelTime = _time;
  386. if (!AudioBuffer.Instance.IsRecording(this))
  387. {
  388. StartRecording();
  389. }
  390. }
  391. // Handle begin recording
  392. private void StartRecording()
  393. {
  394. // Check for input
  395. if (!AudioBuffer.Instance.IsInputAvailable)
  396. {
  397. AudioBuffer.Instance.CheckForInput();
  398. }
  399. // Wait for input and then try again
  400. if (!AudioBuffer.Instance.IsInputAvailable)
  401. {
  402. VoiceEvents.OnError.Invoke("Input Error", "No input source was available. Cannot activate for voice input.");
  403. return;
  404. }
  405. // Already recording
  406. if (AudioBuffer.Instance.IsRecording(this))
  407. {
  408. return;
  409. }
  410. // Start recording
  411. AudioBuffer.Instance.StartRecording(this);
  412. }
  413. // Callback for mic start
  414. private void OnMicStartListening()
  415. {
  416. VoiceEvents?.OnStartListening?.Invoke();
  417. }
  418. // Callback for mic end
  419. private void OnMicStoppedListening()
  420. {
  421. VoiceEvents?.OnStoppedListening?.Invoke();
  422. }
  423. // Callback for mic byte data ready
  424. private void OnByteDataReady(byte[] buffer, int offset, int length)
  425. {
  426. VoiceEvents?.OnByteDataReady.Invoke(buffer, offset, length);
  427. for (int i = 0; null != _dataReadyHandlers && i < _dataReadyHandlers.Length; i++)
  428. {
  429. _dataReadyHandlers[i].OnWitDataReady(buffer, offset, length);
  430. }
  431. }
  432. // Callback for mic sample data ready
  433. private void OnMicSampleReady(RingBuffer<byte>.Marker marker, float levelMax)
  434. {
  435. if (null == _lastSampleMarker || _recordingRequest == null) return;
  436. if (_minSampleByteCount > _lastSampleMarker.RingBuffer.Capacity)
  437. {
  438. _minSampleByteCount = _lastSampleMarker.RingBuffer.Capacity;
  439. }
  440. if (_recordingRequest.State == VoiceRequestState.Transmitting && _recordingRequest.IsInputStreamReady && _lastSampleMarker.AvailableByteCount >= _minSampleByteCount)
  441. {
  442. // Flush the marker since the last read and send it to Wit
  443. _lastSampleMarker.ReadIntoWriters(
  444. (buffer, offset, length) =>
  445. {
  446. _recordingRequest.Write(buffer, offset, length);
  447. },
  448. (buffer, offset, length) => VoiceEvents?.OnByteDataSent?.Invoke(buffer, offset, length),
  449. (buffer, offset, length) =>
  450. {
  451. for (int i = 0; i < _dataSentHandlers.Length; i++)
  452. {
  453. _dataSentHandlers[i]?.OnWitDataSent(buffer, offset, length);
  454. }
  455. });
  456. if (_receivedTranscription)
  457. {
  458. float elapsed = _time - _lastWordTime;
  459. if (elapsed >
  460. RuntimeConfiguration.minTranscriptionKeepAliveTimeInSeconds)
  461. {
  462. VLog.D($"Deactivated due to inactivity. No new words detected in {elapsed:0.00} seconds.");
  463. DeactivateRequest(VoiceEvents?.OnStoppedListeningDueToInactivity);
  464. }
  465. }
  466. else
  467. {
  468. float elapsed = _time - _lastMinVolumeLevelTime;
  469. if (elapsed >
  470. RuntimeConfiguration.minKeepAliveTimeInSeconds)
  471. {
  472. VLog.D($"Deactivated due to inactivity. No sound detected in {elapsed:0.00} seconds.");
  473. DeactivateRequest(VoiceEvents?.OnStoppedListeningDueToInactivity);
  474. }
  475. }
  476. }
  477. else if (_isSoundWakeActive && levelMax > RuntimeConfiguration.soundWakeThreshold)
  478. {
  479. VoiceEvents?.OnMinimumWakeThresholdHit?.Invoke();
  480. SendRecordingRequest();
  481. _lastSampleMarker.Offset(RuntimeConfiguration.sampleLengthInMs * -2);
  482. }
  483. }
  484. // Time tracking for multi-threaded callbacks
  485. private void Update()
  486. {
  487. _time = Time.time;
  488. }
  489. // Mic level change
  490. private void OnMicLevelChanged(float level)
  491. {
  492. if (null != TranscriptionProvider && TranscriptionProvider.OverrideMicLevel) return;
  493. if (level > RuntimeConfiguration.minKeepAliveVolume)
  494. {
  495. _lastMinVolumeLevelTime = _time;
  496. _minKeepAliveWasHit = true;
  497. }
  498. VoiceEvents?.OnMicLevelChanged?.Invoke(level);
  499. }
  500. // Mic level changed in transcription
  501. private void OnTranscriptionMicLevelChanged(float level)
  502. {
  503. if (null != TranscriptionProvider && TranscriptionProvider.OverrideMicLevel)
  504. {
  505. OnMicLevelChanged(level);
  506. }
  507. }
  508. // AudioDurationTracker
  509. private void FinalizeAudioDurationTracker()
  510. {
  511. AudioDurationTracker audioDurationTracker = _recordingRequest?.audioDurationTracker;
  512. if (audioDurationTracker == null)
  513. {
  514. return;
  515. }
  516. if (null == _recordingRequest)
  517. {
  518. VLog.W($"Missing request for recording.");
  519. return;
  520. }
  521. string requestId = _recordingRequest.Options?.RequestId;
  522. if (!string.Equals(requestId, audioDurationTracker.GetRequestId()))
  523. {
  524. VLog.W($"Mismatch in request IDs when finalizing AudioDurationTracker. " +
  525. $"Expected {requestId} but got {audioDurationTracker.GetRequestId()}");
  526. return;
  527. }
  528. audioDurationTracker.FinalizeAudio();
  529. TelemetryEvents.OnAudioTrackerFinished?.Invoke(audioDurationTracker.GetFinalizeTimeStamp(), audioDurationTracker.GetAudioDuration());
  530. }
  531. #endregion
  532. #region DEACTIVATION
  533. /// <summary>
  534. /// Stop listening and submit the collected microphone data to wit for processing.
  535. /// </summary>
  536. public void Deactivate()
  537. {
  538. DeactivateRequest(AudioBuffer.Instance.IsRecording(this) ? VoiceEvents?.OnStoppedListeningDueToDeactivation : null, false);
  539. }
  540. /// <summary>
  541. /// Stop listening and cancel a specific report
  542. /// </summary>
  543. public void DeactivateAndAbortRequest(VoiceServiceRequest request)
  544. {
  545. if (request != null)
  546. {
  547. VoiceEvents?.OnAborting?.Invoke();
  548. request.Cancel();
  549. }
  550. }
  551. /// <summary>
  552. /// Stop listening and abort any requests that may be active without waiting for a response.
  553. /// </summary>
  554. public void DeactivateAndAbortRequest()
  555. {
  556. DeactivateRequest(AudioBuffer.Instance.IsRecording(this) ? VoiceEvents?.OnStoppedListeningDueToDeactivation : null, true);
  557. }
  558. // Stop listening if time expires
  559. private IEnumerator DeactivateDueToTimeLimit()
  560. {
  561. yield return new WaitForSeconds(RuntimeConfiguration.maxRecordingTime);
  562. if (IsRequestActive)
  563. {
  564. VLog.D($"Deactivated input due to timeout.\nMax Record Time: {RuntimeConfiguration.maxRecordingTime}");
  565. DeactivateRequest(VoiceEvents?.OnStoppedListeningDueToTimeout, false);
  566. }
  567. }
  568. private void DeactivateRequest(UnityEvent onComplete = null, bool abort = false)
  569. {
  570. // Aborting
  571. if (abort)
  572. {
  573. VoiceEvents?.OnAborting?.Invoke();
  574. }
  575. // Stop timeout coroutine
  576. if (null != _timeLimitCoroutine)
  577. {
  578. StopCoroutine(_timeLimitCoroutine);
  579. _timeLimitCoroutine = null;
  580. }
  581. // No longer active
  582. _isActive = false;
  583. // Stop recording
  584. StopRecording();
  585. FinalizeAudioDurationTracker();
  586. // Deactivate transcription provider
  587. _activeTranscriptionProvider?.Deactivate();
  588. // Deactivate recording request
  589. WitRequest previousRequest = _recordingRequest;
  590. _recordingRequest = null;
  591. DeactivateWitRequest(previousRequest, abort);
  592. // Abort transmitting requests
  593. if (abort)
  594. {
  595. HashSet<VoiceServiceRequest> requests = _transmitRequests;
  596. _transmitRequests = new HashSet<VoiceServiceRequest>();
  597. foreach (var request in requests)
  598. {
  599. DeactivateWitRequest(request, true);
  600. }
  601. }
  602. // Transmit recording request
  603. else if (previousRequest != null && previousRequest.IsActive && _minKeepAliveWasHit)
  604. {
  605. _transmitRequests.Add(_recordingRequest);
  606. _recordingRequest = null;
  607. VoiceEvents?.OnMicDataSent?.Invoke();
  608. }
  609. // Disable below event
  610. _minKeepAliveWasHit = false;
  611. // Perform on complete event
  612. onComplete?.Invoke();
  613. }
  614. // Deactivate wit request
  615. private void DeactivateWitRequest(VoiceServiceRequest request, bool abort)
  616. {
  617. if (request == null)
  618. {
  619. return;
  620. }
  621. if (abort)
  622. {
  623. request.Cancel("Request was aborted by user.");
  624. }
  625. else
  626. {
  627. request.DeactivateAudio();
  628. }
  629. }
  630. #endregion
  631. #region TRANSCRIPTION
  632. private void OnPartialTranscription(string transcription)
  633. {
  634. _receivedTranscription = true;
  635. _lastWordTime = _time;
  636. VoiceEvents?.OnPartialTranscription.Invoke(transcription);
  637. }
  638. private void OnFullTranscription(string transcription)
  639. {
  640. VoiceEvents?.OnFullTranscription?.Invoke(transcription);
  641. }
  642. #endregion
  643. #region RESPONSE
  644. /// <summary>
  645. /// Main thread call to handle partial response callbacks
  646. /// </summary>
  647. private void HandlePartialResult(WitResponseNode response)
  648. {
  649. if (response != null)
  650. {
  651. VoiceEvents?.OnPartialResponse?.Invoke(response);
  652. }
  653. }
  654. /// <summary>
  655. /// Main thread call to handle result callbacks
  656. /// </summary>
  657. private void HandleResult(VoiceServiceRequest request)
  658. {
  659. // If result is obtained before transcription
  660. if (request == _recordingRequest)
  661. {
  662. DeactivateRequest(null, false);
  663. }
  664. // Handle Success
  665. if (request.State == VoiceRequestState.Successful)
  666. {
  667. VLog.D("Request Success");
  668. VoiceEvents?.OnResponse?.Invoke(request.Results.ResponseData);
  669. VoiceEvents?.OnRequestCompleted?.Invoke();
  670. }
  671. // Handle Cancellation
  672. else if (request.State == VoiceRequestState.Canceled)
  673. {
  674. VLog.D($"Request Canceled\nReason: {request.Results.Message}");
  675. VoiceEvents?.OnCanceled?.Invoke(request.Results.Message);
  676. if (!string.Equals(request.Results.Message, WitConstants.CANCEL_MESSAGE_PRE_SEND))
  677. {
  678. VoiceEvents?.OnAborted?.Invoke();
  679. }
  680. }
  681. // Handle Failure
  682. else if (request.State == VoiceRequestState.Failed)
  683. {
  684. VLog.D($"Request Failed\nError: {request.Results.Message}");
  685. VoiceEvents?.OnError?.Invoke("HTTP Error " + request.Results.StatusCode, request.Results.Message);
  686. VoiceEvents?.OnRequestCompleted?.Invoke();
  687. }
  688. // Remove from transmit list, missing if aborted
  689. if ( _transmitRequests.Contains(request))
  690. {
  691. _transmitRequests.Remove(request);
  692. }
  693. }
  694. /// <summary>
  695. /// Handle request completion
  696. /// </summary>
  697. private void HandleComplete(VoiceServiceRequest request)
  698. {
  699. VoiceEvents?.OnComplete?.Invoke(request);
  700. }
  701. #endregion
  702. }
  703. public interface IWitRuntimeConfigProvider
  704. {
  705. WitRuntimeConfiguration RuntimeConfiguration { get; }
  706. }
  707. public interface IVoiceEventProvider
  708. {
  709. VoiceEvents VoiceEvents { get; }
  710. }
  711. public interface ITelemetryEventsProvider
  712. {
  713. TelemetryEvents TelemetryEvents { get; }
  714. }
  715. }