Api support for UE communication

Frontend/library/src/PixelStreaming/PixelStreaming.ts

@@ -53,6 +53,7 @@ export class PixelStreaming {
     private _videoElementParent: HTMLElement;
 
     _showActionOrErrorOnDisconnect = true;
+    private allowConsoleCommands = false;
 
     private onScreenKeyboardHelper: OnScreenKeyboard;
 
@@ -462,9 +463,9 @@ export class PixelStreaming {
             new InitialSettingsEvent({ settings })
         );
         if (settings.PixelStreamingSettings) {
-            const allowConsoleCommands =
-                settings.PixelStreamingSettings.AllowPixelStreamingCommands;
-            if (allowConsoleCommands === false) {
+            this.allowConsoleCommands =
+                settings.PixelStreamingSettings.AllowPixelStreamingCommands ?? false;
+            if (this.allowConsoleCommands === false) {
                 Logger.Info(
                     Logger.GetStackTrace(),
                     '-AllowPixelStreamingCommands=false, sending arbitrary console commands from browser to UE is disabled.'
@@ -564,6 +565,68 @@ export class PixelStreaming {
         return true;
     }
 
+    /**
+     * Send data to UE application. The data will be run through JSON.stringify() so e.g. strings
+     * and any serializable plain JSON objects with no recurrence can be sent.
+     * @returns true if succeeded, false if rejected
+     */
+    public emitUIInteraction(descriptor: object | string) {
+        if (!this._webRtcController.videoPlayer.isVideoReady()) {
+            return false;
+        }
+        this._webRtcController.emitUIInteraction(descriptor);
+        return true;
+    }
+
+    /**
+     * Send a command to UE application. Blocks ConsoleCommand descriptors unless UE
+     * has signaled that it allows console commands.
+     * @returns true if succeeded, false if rejected
+     */
+    public emitCommand(descriptor: object) {
+        if (!this._webRtcController.videoPlayer.isVideoReady()) {
+            return false;
+        }
+        if (!this.allowConsoleCommands && 'ConsoleCommand' in descriptor) {
+            return false;
+        }
+        this._webRtcController.emitCommand(descriptor);
+        return true;
+    }
+
+    /**
+     * Send a console command to UE application. Only allowed if UE has signaled that it allows
+     * console commands.
+     * @returns true if succeeded, false if rejected
+     */
+    public emitConsoleCommand(command: string) {
+        if (!this.allowConsoleCommands || !this._webRtcController.videoPlayer.isVideoReady()) {
+            return false;
+        }
+        this._webRtcController.emitConsoleCommand(command);
+        return true;
+    }
+
+    /**
+     * Add a UE -> browser response event listener
+     * @param name - The name of the response handler
+     * @param listener - The method to be activated when a message is received
+     */
+    public addResponseEventListener(
+        name: string,
+        listener: (response: string) => void
+    ) {
+        this._webRtcController.responseController.addResponseEventListener(name, listener);
+    }
+
+    /**
+     * Remove a UE -> browser response event listener
+     * @param name - The name of the response handler
+     */
+    public removeResponseEventListener(name: string) {
+        this._webRtcController.responseController.removeResponseEventListener(name);
+    }
+
     /**
      * Dispatch a new event.
      * @param e event

Frontend/library/src/UeInstanceMessage/SendDescriptorController.ts

@@ -36,7 +36,7 @@ export class SendDescriptorController {
      * Send a Latency Test to the UE Instance
      * @param descriptor - the descriptor for a UI Interaction
      */
-    emitUIInteraction(descriptor: object) {
+    emitUIInteraction(descriptor: object | string) {
         this.sendDescriptor('UIInteraction', descriptor);
     }
 
@@ -45,7 +45,7 @@ export class SendDescriptorController {
      * @param messageType - UE Message Type
      * @param descriptor - Descriptor Message as JSON
      */
-    sendDescriptor(messageType: string, descriptor: object) {
+    sendDescriptor(messageType: string, descriptor: object | string) {
         // Convert the descriptor object into a JSON string.
         const descriptorAsString = JSON.stringify(descriptor);
         const toStreamerMessages =

Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts

@@ -1671,6 +1671,44 @@ export class WebRtcPlayerController {
         this.streamMessageController.toStreamerHandlers.get('IFrameRequest')();
     }
 
+    /**
+     * Send a UIInteraction message
+     */
+    emitUIInteraction(descriptor: object | string) {
+        Logger.Log(
+            Logger.GetStackTrace(),
+            '----   Sending custom UIInteraction message   ----',
+            6
+        );
+        this.sendDescriptorController.emitUIInteraction(descriptor);
+    }
+
+    /**
+     * Send a Command message
+     */
+    emitCommand(descriptor: object) {
+        Logger.Log(
+            Logger.GetStackTrace(),
+            '----   Sending custom Command message   ----',
+            6
+        );
+        this.sendDescriptorController.emitCommand(descriptor);
+    }
+
+    /**
+     * Send a console command message
+     */
+    emitConsoleCommand(command: string) {
+        Logger.Log(
+            Logger.GetStackTrace(),
+            '----   Sending custom Command:ConsoleCommand message   ----',
+            6
+        );
+        this.sendDescriptorController.emitCommand({
+            ConsoleCommand: command,
+        });
+    }
+
     /**
      * Sends a request to the UE Instance to have ownership of Quality
      */