WitRequest.cs 36 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024
  1. /*
  2. * Copyright (c) Meta Platforms, Inc. and affiliates.
  3. * All rights reserved.
  4. *
  5. * This source code is licensed under the license found in the
  6. * LICENSE file in the root directory of this source tree.
  7. */
  8. using System;
  9. using System.Collections.Concurrent;
  10. using System.Collections.Generic;
  11. using System.IO;
  12. using System.Linq;
  13. using System.Net;
  14. using System.Text;
  15. using System.Threading;
  16. using Meta.Voice;
  17. using Meta.WitAi.Configuration;
  18. using Meta.WitAi.Data;
  19. using Meta.WitAi.Data.Configuration;
  20. using Meta.WitAi.Json;
  21. using Meta.WitAi.Requests;
  22. using UnityEngine;
  23. using UnityEngine.Networking;
  24. #if UNITY_EDITOR
  25. using UnityEditor;
  26. #endif
  27. namespace Meta.WitAi
  28. {
  29. /// <summary>
  30. /// Manages a single request lifecycle when sending/receiving data from Wit.ai.
  31. ///
  32. /// Note: This is not intended to be instantiated directly. Requests should be created with the
  33. /// WitRequestFactory
  34. /// </summary>
  35. public class WitRequest : VoiceServiceRequest
  36. {
  37. #region PARAMETERS
  38. /// <summary>
  39. /// The wit Configuration to be used with this request
  40. /// </summary>
  41. public WitConfiguration Configuration { get; private set; }
  42. /// <summary>
  43. /// The request timeout in ms
  44. /// </summary>
  45. public int Timeout { get; private set; } = 1000;
  46. /// <summary>
  47. /// Encoding settings for audio based requests
  48. /// </summary>
  49. public AudioEncoding AudioEncoding { get; set; }
  50. [Obsolete("Deprecated for AudioEncoding")]
  51. public AudioEncoding audioEncoding
  52. {
  53. get => AudioEncoding;
  54. set => AudioEncoding = value;
  55. }
  56. /// <summary>
  57. /// Endpoint to be used for this request
  58. /// </summary>
  59. public string Path { get; private set; }
  60. /// <summary>
  61. /// Final portion of the endpoint Path
  62. /// </summary>
  63. public string Command { get; private set; }
  64. /// <summary>
  65. /// Whether a post command should be called
  66. /// </summary>
  67. public bool IsPost { get; private set; }
  68. /// <summary>
  69. /// Key value pair that is sent as a query param in the Wit.ai uri
  70. /// </summary>
  71. [Obsolete("Deprecated for Options.QueryParams")]
  72. public VoiceServiceRequestOptions.QueryParam[] queryParams
  73. {
  74. get
  75. {
  76. List<VoiceServiceRequestOptions.QueryParam> results = new List<VoiceServiceRequestOptions.QueryParam>();
  77. foreach (var key in Options?.QueryParams?.Keys)
  78. {
  79. VoiceServiceRequestOptions.QueryParam p = new VoiceServiceRequestOptions.QueryParam()
  80. {
  81. key = key,
  82. value = Options?.QueryParams[key]
  83. };
  84. results.Add(p);
  85. }
  86. return results.ToArray();
  87. }
  88. }
  89. public byte[] postData;
  90. public string postContentType;
  91. public string forcedHttpMethodType = null;
  92. #endregion PARAMETERS
  93. #region REQUEST
  94. /// <summary>
  95. /// Returns true if the request is being performed
  96. /// </summary>
  97. public bool IsRequestStreamActive => IsActive || IsInputStreamReady;
  98. /// <summary>
  99. /// Returns true if the response had begun
  100. /// </summary>
  101. public bool HasResponseStarted { get; private set; }
  102. /// <summary>
  103. /// Returns true if the response had begun
  104. /// </summary>
  105. public bool IsInputStreamReady { get; private set; }
  106. public AudioDurationTracker audioDurationTracker;
  107. private HttpWebRequest _request;
  108. private Stream _writeStream;
  109. private object _streamLock = new object();
  110. private int _bytesWritten;
  111. private string _stackTrace;
  112. private DateTime _requestStartTime;
  113. private ConcurrentQueue<byte[]> _writeBuffer = new ConcurrentQueue<byte[]>();
  114. #endregion REQUEST
  115. #region RESULTS
  116. /// <summary>
  117. /// The current status of the request
  118. /// </summary>
  119. public string StatusDescription { get; private set; }
  120. /// <summary>
  121. /// Simply return the Path to be called
  122. /// </summary>
  123. public override string ToString() => Path;
  124. /// <summary>
  125. /// Last response data parsed
  126. /// </summary>
  127. private WitResponseNode _lastResponseData;
  128. #endregion RESULTS
  129. #region EVENTS
  130. /// <summary>
  131. /// Provides an opportunity to provide custom headers for the request just before it is
  132. /// executed.
  133. /// </summary>
  134. public event OnProvideCustomHeadersEvent onProvideCustomHeaders;
  135. public delegate Dictionary<string, string> OnProvideCustomHeadersEvent();
  136. /// <summary>
  137. /// Callback called when the server is ready to receive data from the WitRequest's input
  138. /// stream. See WitRequest.Write()
  139. /// </summary>
  140. public event Action<WitRequest> onInputStreamReady;
  141. /// <summary>
  142. /// Returns the raw string response that was received before converting it to a JSON object.
  143. ///
  144. /// NOTE: This response comes back on a different thread. Do not attempt ot set UI control
  145. /// values or other interactions from this callback. This is intended to be used for demo
  146. /// and test UI, not for regular use.
  147. /// </summary>
  148. public Action<string> onRawResponse;
  149. /// <summary>
  150. /// Provides an opportunity to customize the url just before a request executed
  151. /// </summary>
  152. [Obsolete("Deprecated for WitVRequest.OnProvideCustomUri")]
  153. public OnCustomizeUriEvent onCustomizeUri;
  154. public delegate Uri OnCustomizeUriEvent(UriBuilder uriBuilder);
  155. /// <summary>
  156. /// Allows customization of the request before it is sent out.
  157. ///
  158. /// Note: This is for devs who are routing requests to their servers
  159. /// before sending data to Wit.ai. This allows adding any additional
  160. /// headers, url modifications, or customization of the request.
  161. /// </summary>
  162. public static PreSendRequestDelegate onPreSendRequest;
  163. public delegate void PreSendRequestDelegate(ref Uri src_uri, out Dictionary<string,string> headers);
  164. /// <summary>
  165. /// Returns a partial utterance from an in process request
  166. ///
  167. /// NOTE: This response comes back on a different thread.
  168. /// </summary>
  169. [Obsolete("Deprecated for Events.OnPartialTranscription")]
  170. public event Action<string> onPartialTranscription;
  171. /// <summary>
  172. /// Returns a full utterance from a completed request
  173. ///
  174. /// NOTE: This response comes back on a different thread.
  175. /// </summary>
  176. [Obsolete("Deprecated for Events.OnFullTranscription")]
  177. public event Action<string> onFullTranscription;
  178. /// <summary>
  179. /// Callback called when a response is received from the server off a partial transcription
  180. /// </summary>
  181. [Obsolete("Deprecated for Events.OnPartialResponse")]
  182. public event Action<WitRequest> onPartialResponse;
  183. /// <summary>
  184. /// Callback called when a response is received from the server
  185. /// </summary>
  186. [Obsolete("Deprecated for Events.OnComplete")]
  187. public event Action<WitRequest> onResponse;
  188. #endregion EVENTS
  189. #region INITIALIZATION
  190. /// <summary>
  191. /// Initialize wit request with configuration & path to endpoint
  192. /// </summary>
  193. /// <param name="newConfiguration"></param>
  194. /// <param name="newOptions"></param>
  195. /// <param name="newEvents"></param>
  196. public WitRequest(WitConfiguration newConfiguration, string newPath,
  197. WitRequestOptions newOptions, VoiceServiceRequestEvents newEvents)
  198. : base(NLPRequestInputType.Audio, newOptions, newEvents)
  199. {
  200. // Set Configuration & path
  201. Configuration = newConfiguration;
  202. Path = newPath;
  203. // Finalize
  204. _initialized = true;
  205. SetState(VoiceRequestState.Initialized);
  206. }
  207. /// <summary>
  208. /// Only set state if initialized
  209. /// </summary>
  210. private bool _initialized = false;
  211. protected override void SetState(VoiceRequestState newState)
  212. {
  213. if (_initialized)
  214. {
  215. base.SetState(newState);
  216. }
  217. }
  218. /// <summary>
  219. /// Finalize initialization
  220. /// </summary>
  221. protected override void OnInit()
  222. {
  223. // Determine configuration setting
  224. Timeout = Configuration == null ? Timeout : Configuration.timeoutMS;
  225. // Set request settings
  226. Command = Path.Split('/').First();
  227. IsPost = WitEndpointConfig.GetEndpointConfig(Configuration).Speech == this.Command
  228. || WitEndpointConfig.GetEndpointConfig(Configuration).Dictation == this.Command;
  229. // Finalize bases
  230. base.OnInit();
  231. }
  232. #endregion INITIALIZATION
  233. #region AUDIO
  234. // Handle audio activation
  235. protected override void HandleAudioActivation()
  236. {
  237. SetAudioInputState(VoiceAudioInputState.On);
  238. }
  239. // Handle audio deactivation
  240. protected override void HandleAudioDeactivation()
  241. {
  242. // If transmitting,
  243. if (State == VoiceRequestState.Transmitting)
  244. {
  245. CloseRequestStream();
  246. }
  247. // Call deactivated
  248. SetAudioInputState(VoiceAudioInputState.Off);
  249. }
  250. #endregion
  251. #region REQUEST
  252. // Errors that prevent request submission
  253. protected override string GetSendError()
  254. {
  255. // No configuration found
  256. if (Configuration == null)
  257. {
  258. return "Configuration is not set. Cannot start request.";
  259. }
  260. // Cannot start without client access token
  261. if (string.IsNullOrEmpty(Configuration.GetClientAccessToken()))
  262. {
  263. return "Client access token is not defined. Cannot start request.";
  264. }
  265. // Cannot perform without input stream delegate
  266. if (onInputStreamReady == null)
  267. {
  268. return "No input stream delegate found";
  269. }
  270. // Base
  271. return base.GetSendError();
  272. }
  273. // Simple getter for final uri
  274. private Uri GetUri()
  275. {
  276. // Get query parameters
  277. Dictionary<string, string> queryParams = new Dictionary<string, string>(Options.QueryParams);
  278. // Get uri using override
  279. var uri = WitVRequest.GetWitUri(Configuration, Path, queryParams);
  280. #pragma warning disable CS0618
  281. if (onCustomizeUri != null)
  282. {
  283. #pragma warning disable CS0618
  284. uri = onCustomizeUri(new UriBuilder(uri));
  285. }
  286. // Return uri
  287. return uri;
  288. }
  289. // Simple getter for final uri
  290. private Dictionary<string, string> GetHeaders()
  291. {
  292. // Get default headers
  293. Dictionary<string, string> headers = WitVRequest.GetWitHeaders(Configuration, Options?.RequestId, false);
  294. // Append additional headers
  295. if (onProvideCustomHeaders != null)
  296. {
  297. foreach (OnProvideCustomHeadersEvent e in onProvideCustomHeaders.GetInvocationList())
  298. {
  299. Dictionary<string, string> customHeaders = e();
  300. if (customHeaders != null)
  301. {
  302. foreach (var key in customHeaders.Keys)
  303. {
  304. headers[key] = customHeaders[key];
  305. }
  306. }
  307. }
  308. }
  309. // Return headers
  310. return headers;
  311. }
  312. /// <summary>
  313. /// Start the async request for data from the Wit.ai servers
  314. /// </summary>
  315. protected override void HandleSend()
  316. {
  317. // Begin
  318. HasResponseStarted = false;
  319. // Generate results
  320. StatusCode = 0;
  321. StatusDescription = "Starting request";
  322. _bytesWritten = 0;
  323. _requestStartTime = DateTime.UtcNow;
  324. _stackTrace = "-";
  325. // Get uri & headers
  326. var uri = GetUri();
  327. var headers = GetHeaders();
  328. // Allow overrides
  329. onPreSendRequest?.Invoke(ref uri, out headers);
  330. #if UNITY_WEBGL && !UNITY_EDITOR
  331. StartUnityRequest(uri, headers);
  332. #else
  333. #if UNITY_WEBGL && UNITY_EDITOR
  334. if (IsPost)
  335. {
  336. VLog.W("Voice input is not supported in WebGL this functionality is fully enabled at edit time, but may not work at runtime.");
  337. }
  338. #endif
  339. StartThreadedRequest(uri, headers);
  340. #endif
  341. }
  342. #endregion REQUEST
  343. #region HTTP REQUEST
  344. /// <summary>
  345. /// Performs a threaded http request
  346. /// </summary>
  347. private void StartThreadedRequest(Uri uri, Dictionary<string, string> headers)
  348. {
  349. // Create http web request
  350. _request = WebRequest.Create(uri.AbsoluteUri) as HttpWebRequest;
  351. // Off to not wait for a response indefinitely
  352. _request.KeepAlive = false;
  353. // Configure request method, content type & chunked
  354. if (forcedHttpMethodType != null)
  355. {
  356. _request.Method = forcedHttpMethodType;
  357. }
  358. if (null != postContentType)
  359. {
  360. if (forcedHttpMethodType == null) {
  361. _request.Method = "POST";
  362. }
  363. _request.ContentType = postContentType;
  364. _request.ContentLength = postData.Length;
  365. }
  366. if (IsPost)
  367. {
  368. _request.Method = string.IsNullOrEmpty(forcedHttpMethodType) ? "POST" : forcedHttpMethodType;
  369. _request.ContentType = AudioEncoding.ToString();
  370. _request.SendChunked = true;
  371. }
  372. // Apply user agent
  373. if (headers.ContainsKey(WitConstants.HEADER_USERAGENT))
  374. {
  375. _request.UserAgent = headers[WitConstants.HEADER_USERAGENT];
  376. headers.Remove(WitConstants.HEADER_USERAGENT);
  377. }
  378. // Apply all other headers
  379. foreach (var key in headers.Keys)
  380. {
  381. _request.Headers[key] = headers[key];
  382. }
  383. // Apply timeout
  384. _request.Timeout = Timeout;
  385. // Begin calling on main thread if needed
  386. WatchMainThreadCallbacks();
  387. // Perform http post or put
  388. if (_request.Method == "POST" || _request.Method == "PUT")
  389. {
  390. var getRequestTask = _request.BeginGetRequestStream(HandleWriteStream, _request);
  391. ThreadPool.RegisterWaitForSingleObject(getRequestTask.AsyncWaitHandle,
  392. HandleTimeoutTimer, _request, Timeout, true);
  393. }
  394. // Move right to response
  395. else
  396. {
  397. StartResponse();
  398. }
  399. }
  400. // Start response
  401. private void StartResponse()
  402. {
  403. if (_request == null)
  404. {
  405. if (StatusCode == 0)
  406. {
  407. StatusCode = WitConstants.ERROR_CODE_GENERAL;
  408. StatusDescription = $"Request canceled prior to start";
  409. }
  410. HandleNlpResponse(null, StatusDescription);
  411. return;
  412. }
  413. var asyncResult = _request.BeginGetResponse(HandleResponse, _request);
  414. ThreadPool.RegisterWaitForSingleObject(asyncResult.AsyncWaitHandle, HandleTimeoutTimer, _request, Timeout, true);
  415. }
  416. // Handle timeout callback
  417. private void HandleTimeoutTimer(object state, bool timeout)
  418. {
  419. // Ignore false or too late
  420. if (!timeout)
  421. {
  422. return;
  423. }
  424. // No longer active
  425. StatusCode = WitConstants.ERROR_CODE_TIMEOUT;
  426. StatusDescription = $"Request timed out after {(DateTime.UtcNow - _requestStartTime).Seconds:0.00} seconds";
  427. // Clean up the current request if it is still going
  428. if (null != _request)
  429. {
  430. _request.Abort();
  431. }
  432. // Close any open stream resources and clean up streaming state flags
  433. CloseActiveStream();
  434. // Complete
  435. MainThreadCallback(() => HandleNlpResponse(null, StatusDescription));
  436. }
  437. // Write stream
  438. private void HandleWriteStream(IAsyncResult ar)
  439. {
  440. try
  441. {
  442. // Start response stream
  443. StartResponse();
  444. // Get write stream
  445. var stream = _request.EndGetRequestStream(ar);
  446. // Got write stream
  447. _bytesWritten = 0;
  448. // Immediate post
  449. if (postData != null && postData.Length > 0)
  450. {
  451. Debug.Log("Wrote directly");
  452. _bytesWritten += postData.Length;
  453. stream.Write(postData, 0, postData.Length);
  454. stream.Close();
  455. }
  456. // Wait for input stream
  457. else
  458. {
  459. // Request stream is ready to go
  460. IsInputStreamReady = true;
  461. _writeStream = stream;
  462. // Call input stream ready delegate
  463. if (onInputStreamReady != null)
  464. {
  465. MainThreadCallback(() => onInputStreamReady(this));
  466. }
  467. }
  468. }
  469. catch (WebException e)
  470. {
  471. // Ignore cancelation errors & if error already occured
  472. if (e.Status == WebExceptionStatus.RequestCanceled || StatusCode != 0)
  473. {
  474. return;
  475. }
  476. // Write stream error
  477. _stackTrace = e.StackTrace;
  478. StatusCode = (int) e.Status;
  479. StatusDescription = e.Message;
  480. VLog.W(e);
  481. MainThreadCallback(() => HandleNlpResponse(null, StatusDescription));
  482. }
  483. catch (Exception e)
  484. {
  485. // Call an error if have not done so yet
  486. if (StatusCode != 0)
  487. {
  488. return;
  489. }
  490. // Non web error occured
  491. _stackTrace = e.StackTrace;
  492. StatusCode = WitConstants.ERROR_CODE_GENERAL;
  493. StatusDescription = e.Message;
  494. VLog.W(e);
  495. MainThreadCallback(() => HandleNlpResponse(null, StatusDescription));
  496. }
  497. }
  498. /// <summary>
  499. /// Write request data to the Wit.ai post's body input stream
  500. ///
  501. /// Note: If the stream is not open (IsActive) this will throw an IOException.
  502. /// Data will be written synchronously. This should not be called from the main thread.
  503. /// </summary>
  504. /// <param name="data"></param>
  505. /// <param name="offset"></param>
  506. /// <param name="length"></param>
  507. public void Write(byte[] data, int offset, int length)
  508. {
  509. // Ignore without write stream
  510. if (!IsInputStreamReady || data == null || length == 0)
  511. {
  512. return;
  513. }
  514. try
  515. {
  516. _writeStream.Write(data, offset, length);
  517. _bytesWritten += length;
  518. if (audioDurationTracker != null)
  519. {
  520. audioDurationTracker.AddBytes(length);
  521. }
  522. }
  523. catch (ObjectDisposedException e)
  524. {
  525. // Handling edge case where stream is closed remotely
  526. // This problem occurs when the Web server resets or closes the connection after
  527. // the client application sends the HTTP header.
  528. // https://support.microsoft.com/en-us/topic/fix-you-receive-a-system-objectdisposedexception-exception-when-you-try-to-access-a-stream-object-that-is-returned-by-the-endgetrequeststream-method-in-the-net-framework-2-0-bccefe57-0a61-517a-5d5f-2dce0cc63265
  529. VLog.W($"Stream already disposed. It is likely the server reset the connection before streaming started.\n{e}");
  530. // This prevents a very long holdup on _writeStream.Close
  531. _writeStream = null;
  532. }
  533. catch (IOException e)
  534. {
  535. VLog.W(e.Message);
  536. }
  537. catch (Exception e)
  538. {
  539. VLog.E(e);
  540. }
  541. // Perform a cancellation if still waiting for a post
  542. if (WaitingForPost())
  543. {
  544. MainThreadCallback(() => Cancel("Stream was closed with no data written."));
  545. }
  546. }
  547. // Handles response from server
  548. private void HandleResponse(IAsyncResult asyncResult)
  549. {
  550. // Begin response
  551. HasResponseStarted = true;
  552. string stringResponse = "";
  553. try
  554. {
  555. // Get response
  556. CheckStatus();
  557. using (var response = _request.EndGetResponse(asyncResult))
  558. {
  559. // Got response
  560. CheckStatus();
  561. HttpWebResponse httpResponse = response as HttpWebResponse;
  562. // Apply status & description
  563. StatusCode = (int) httpResponse.StatusCode;
  564. StatusDescription = httpResponse.StatusDescription;
  565. // Get stream
  566. using (var responseStream = httpResponse.GetResponseStream())
  567. {
  568. using (var responseReader = new StreamReader(responseStream))
  569. {
  570. string chunk;
  571. while ((chunk = ReadToDelimiter(responseReader, WitConstants.ENDPOINT_JSON_DELIMITER)) != null)
  572. {
  573. stringResponse = chunk;
  574. ProcessStringResponse(stringResponse);
  575. }
  576. }
  577. }
  578. }
  579. }
  580. catch (JSONParseException e)
  581. {
  582. _stackTrace = e.StackTrace;
  583. StatusCode = WitConstants.ERROR_CODE_INVALID_DATA_FROM_SERVER;
  584. StatusDescription = "Server returned invalid data.";
  585. VLog.W(e);
  586. }
  587. catch (WebException e)
  588. {
  589. if (e.Status != WebExceptionStatus.RequestCanceled)
  590. {
  591. // Apply status & error
  592. _stackTrace = e.StackTrace;
  593. StatusCode = (int) e.Status;
  594. StatusDescription = e.Message;
  595. VLog.W(e);
  596. // Attempt additional parse
  597. if (e.Response is HttpWebResponse errorResponse)
  598. {
  599. StatusCode = (int) errorResponse.StatusCode;
  600. try
  601. {
  602. using (var errorStream = errorResponse.GetResponseStream())
  603. {
  604. if (errorStream != null)
  605. {
  606. using (StreamReader errorReader = new StreamReader(errorStream))
  607. {
  608. stringResponse = errorReader.ReadToEnd();
  609. if (!string.IsNullOrEmpty(stringResponse))
  610. {
  611. ProcessStringResponses(stringResponse);
  612. }
  613. }
  614. }
  615. }
  616. }
  617. catch (JSONParseException)
  618. {
  619. // Response wasn't encoded error, ignore it.
  620. }
  621. catch (Exception errorResponseError)
  622. {
  623. // We've already caught that there is an error, we'll ignore any errors
  624. // reading error response data and use the status/original error for validation
  625. VLog.W(errorResponseError);
  626. _stackTrace = e.StackTrace;
  627. }
  628. }
  629. }
  630. }
  631. catch (Exception e)
  632. {
  633. _stackTrace = e.StackTrace;
  634. StatusCode = WitConstants.ERROR_CODE_GENERAL;
  635. StatusDescription = e.Message;
  636. VLog.W(e);
  637. }
  638. // Close request stream if possible
  639. CloseRequestStream();
  640. // Confirm valid response
  641. if (null != _lastResponseData)
  642. {
  643. var error = _lastResponseData["error"];
  644. if (!string.IsNullOrEmpty(error))
  645. {
  646. // Get code if possible
  647. var code = _lastResponseData["code"];
  648. if (code != null)
  649. {
  650. StatusCode = code.AsInt;
  651. }
  652. // Use general error if code is not provided
  653. if (StatusCode == (int)HttpStatusCode.OK)
  654. {
  655. StatusCode = WitConstants.ERROR_CODE_GENERAL;
  656. }
  657. // Set error & description
  658. if (string.IsNullOrEmpty(StatusDescription))
  659. {
  660. StatusDescription = $"Error: {code}\n{error}";
  661. }
  662. }
  663. }
  664. // Invalid response
  665. else if (StatusCode == (int)HttpStatusCode.OK)
  666. {
  667. StatusCode = WitConstants.ERROR_CODE_NO_DATA_FROM_SERVER;
  668. StatusDescription = $"Server did not return a valid json response.";
  669. #if UNITY_EDITOR
  670. StatusDescription += $"\nActual Response\n{stringResponse}";
  671. #endif
  672. }
  673. // Done
  674. HasResponseStarted = false;
  675. MainThreadCallback(() =>
  676. {
  677. // Send partial data if not previously sent
  678. if (!_lastResponseData.HasResponse())
  679. {
  680. ResponseData = _lastResponseData;
  681. }
  682. // Apply error if needed
  683. if (null != _lastResponseData)
  684. {
  685. var error = _lastResponseData["error"];
  686. if (!string.IsNullOrEmpty(error))
  687. {
  688. StatusDescription += $"\n{error}";
  689. }
  690. }
  691. // Call completion delegate
  692. HandleNlpResponse(_lastResponseData, StatusCode == (int)HttpStatusCode.OK ? string.Empty : $"{StatusDescription}\n\nStackTrace:\n{_stackTrace}\n\n");
  693. });
  694. }
  695. // Check status
  696. private void CheckStatus()
  697. {
  698. if (StatusCode == 0) return;
  699. switch (StatusCode)
  700. {
  701. case WitConstants.ERROR_CODE_ABORTED:
  702. throw new WebException("Request was aborted", WebExceptionStatus.RequestCanceled);
  703. default:
  704. throw new WebException("Status changed before response was received.", (WebExceptionStatus) StatusCode);
  705. }
  706. }
  707. // Read stream until delimiter is hit
  708. private string ReadToDelimiter(StreamReader reader, string delimiter)
  709. {
  710. // Allocate all vars
  711. StringBuilder results = new StringBuilder();
  712. int delLength = delimiter.Length;
  713. int i;
  714. bool found;
  715. char nextChar;
  716. // Iterate each byte in the stream
  717. while (reader != null && !reader.EndOfStream)
  718. {
  719. // Continue until found
  720. if (reader.Peek() == 0)
  721. {
  722. continue;
  723. }
  724. // Append next character
  725. nextChar = (char)reader.Read();
  726. results.Append(nextChar);
  727. // Continue until long as delimiter
  728. if (results.Length < delLength)
  729. {
  730. continue;
  731. }
  732. // Check if string builder ends with delimiter
  733. found = true;
  734. for (i=0;i<delLength;i++)
  735. {
  736. // Stop checking if not delimiter
  737. if (delimiter[i] != results[results.Length - delLength + i])
  738. {
  739. found = false;
  740. break;
  741. }
  742. }
  743. // Found delimiter
  744. if (found)
  745. {
  746. return results.ToString(0, results.Length - delLength);
  747. }
  748. }
  749. // If no delimiter is found, return the rest of the chunk
  750. return results.Length == 0 ? null : results.ToString();
  751. }
  752. // Process individual piece
  753. private void ProcessStringResponses(string stringResponse)
  754. {
  755. // Split by delimiter
  756. foreach (var stringPart in stringResponse.Split(new string[]{WitConstants.ENDPOINT_JSON_DELIMITER}, StringSplitOptions.RemoveEmptyEntries))
  757. {
  758. ProcessStringResponse(stringPart);
  759. }
  760. }
  761. // Safely handles
  762. private void ProcessStringResponse(string stringResponse)
  763. {
  764. // Call raw response for every received response
  765. if (!string.IsNullOrEmpty(stringResponse))
  766. {
  767. MainThreadCallback(() => onRawResponse?.Invoke(stringResponse));
  768. }
  769. // Decode full response
  770. WitResponseNode responseNode = WitResponseNode.Parse(stringResponse);
  771. bool hasResponse = responseNode.HasResponse();
  772. bool isFinal = responseNode.GetIsFinal();
  773. string transcription = responseNode.GetTranscription();
  774. _lastResponseData = responseNode;
  775. // Apply on main thread
  776. MainThreadCallback(() =>
  777. {
  778. // Set transcription
  779. if (!string.IsNullOrEmpty(transcription) && (!hasResponse || isFinal))
  780. {
  781. ApplyTranscription(transcription, isFinal);
  782. }
  783. // Set response
  784. if (hasResponse)
  785. {
  786. ResponseData = responseNode;
  787. }
  788. });
  789. }
  790. // On text change callback
  791. protected override void OnTranscriptionChanged()
  792. {
  793. if (!IsFinalTranscription)
  794. {
  795. onPartialTranscription?.Invoke(Transcription);
  796. }
  797. else
  798. {
  799. onFullTranscription?.Invoke(Transcription);
  800. }
  801. base.OnTranscriptionChanged();
  802. }
  803. // On response data change callback
  804. protected override void OnResponseDataChanged()
  805. {
  806. onPartialResponse?.Invoke(this);
  807. base.OnResponseDataChanged();
  808. }
  809. // Check if data has been written to post stream while still receiving data
  810. private bool WaitingForPost()
  811. {
  812. return IsPost && _bytesWritten == 0 && StatusCode == 0;
  813. }
  814. // Close active stream & then abort if possible
  815. private void CloseRequestStream()
  816. {
  817. // Cancel due to no audio if not an error
  818. if (WaitingForPost())
  819. {
  820. Cancel("Request was closed with no audio captured.");
  821. }
  822. // Close
  823. else
  824. {
  825. CloseActiveStream();
  826. }
  827. }
  828. // Close stream
  829. private void CloseActiveStream()
  830. {
  831. IsInputStreamReady = false;
  832. lock (_streamLock)
  833. {
  834. if (null != _writeStream)
  835. {
  836. try
  837. {
  838. _writeStream.Close();
  839. }
  840. catch (Exception e)
  841. {
  842. VLog.W($"Write Stream - Close Failed\n{e}");
  843. }
  844. _writeStream = null;
  845. }
  846. }
  847. }
  848. // Perform a cancellation/abort
  849. protected override void HandleCancel()
  850. {
  851. // Close stream
  852. CloseActiveStream();
  853. // Apply abort code
  854. if (StatusCode == 0)
  855. {
  856. StatusCode = WitConstants.ERROR_CODE_ABORTED;
  857. StatusDescription = Results.Message;
  858. }
  859. // Abort request
  860. if (null != _request)
  861. {
  862. _request.Abort();
  863. _request = null;
  864. }
  865. }
  866. // Add response callback & log for abort
  867. protected override void OnComplete()
  868. {
  869. base.OnComplete();
  870. // Close write stream if still existing
  871. if (null != _writeStream)
  872. {
  873. CloseActiveStream();
  874. }
  875. // Abort request if still existing
  876. if (null != _request)
  877. {
  878. _request.Abort();
  879. _request = null;
  880. }
  881. // Finalize response
  882. onResponse?.Invoke(this);
  883. onResponse = null;
  884. }
  885. #endregion HTTP REQUEST
  886. #region CALLBACKS
  887. // Check performing
  888. private CoroutineUtility.CoroutinePerformer _performer = null;
  889. // All actions
  890. private ConcurrentQueue<Action> _mainThreadCallbacks = new ConcurrentQueue<Action>();
  891. // Called from background thread
  892. private void MainThreadCallback(Action action)
  893. {
  894. if (action == null)
  895. {
  896. return;
  897. }
  898. _mainThreadCallbacks.Enqueue(action);
  899. }
  900. // While active, perform any sent callbacks
  901. private void WatchMainThreadCallbacks()
  902. {
  903. // Ignore if already performing
  904. if (_performer != null)
  905. {
  906. return;
  907. }
  908. // Check callbacks every frame (editor or runtime)
  909. _performer = CoroutineUtility.StartCoroutine(PerformMainThreadCallbacks());
  910. }
  911. // Every frame check for callbacks & perform any found
  912. private System.Collections.IEnumerator PerformMainThreadCallbacks()
  913. {
  914. // While checking, continue
  915. while (HasMainThreadCallbacks())
  916. {
  917. // Wait for frame
  918. if (Application.isPlaying && !Application.isBatchMode)
  919. {
  920. yield return new WaitForEndOfFrame();
  921. }
  922. // Wait for a tick
  923. else
  924. {
  925. yield return null;
  926. }
  927. // Perform if possible
  928. while (_mainThreadCallbacks.Count > 0 && _mainThreadCallbacks.TryDequeue(out var result))
  929. {
  930. result();
  931. }
  932. }
  933. _performer = null;
  934. }
  935. // If active or performing callbacks
  936. private bool HasMainThreadCallbacks()
  937. {
  938. return IsActive || _mainThreadCallbacks.Count > 0;
  939. }
  940. #endregion
  941. }
  942. }